2024-04-26T04:25:07.952 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-26T04:25:07.952 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562 --verbose --owner scheduled_teuthology@teuthology --name teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/orig.config.yaml 2024-04-26T04:25:08.013 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T04:25:08.107 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7674562' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 8320 sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 10/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a targets: smithi038.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAacMjVVYK871bpzS385fVCDFelFLOfZinui4I3jyXCbdZW7ex1QlubBdLTLS8M+N6YfN8ezerdLtFpgVkrWcpg= smithi055.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPnZbdTwWwE5etQ8bkwSIvLYlHKEecZ0zgXcLkBgyZngtxPTJd5Z33X0vk0RiST7WJRAWhx9LHhwLJjVniEFw0c= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-26_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-26T04:25:08.107 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa; will attempt to use it 2024-04-26T04:25:08.108 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks 2024-04-26T04:25:08.108 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-26T04:25:08.110 INFO:teuthology.task.internal:Checking packages... 2024-04-26T04:25:08.133 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a' 2024-04-26T04:25:08.133 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:25:08.133 INFO:teuthology.packaging:ref: None 2024-04-26T04:25:08.133 INFO:teuthology.packaging:tag: None 2024-04-26T04:25:08.133 INFO:teuthology.packaging:branch: squid 2024-04-26T04:25:08.134 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:25:08.134 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-26T04:25:08.486 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2447.gb22e2ebd 2024-04-26T04:25:08.488 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-26T04:25:08.494 INFO:teuthology.task.internal:no buildpackages task found 2024-04-26T04:25:08.494 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-26T04:25:08.510 INFO:teuthology.task.internal:Saving configuration 2024-04-26T04:25:08.534 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-26T04:25:08.540 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T04:25:08.566 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi038.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:20:17.323916', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAacMjVVYK871bpzS385fVCDFelFLOfZinui4I3jyXCbdZW7ex1QlubBdLTLS8M+N6YfN8ezerdLtFpgVkrWcpg='} 2024-04-26T04:25:08.591 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:20:17.325037', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPnZbdTwWwE5etQ8bkwSIvLYlHKEecZ0zgXcLkBgyZngtxPTJd5Z33X0vk0RiST7WJRAWhx9LHhwLJjVniEFw0c='} 2024-04-26T04:25:08.591 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-26T04:25:08.597 INFO:teuthology.task.internal:roles: ubuntu@smithi038.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-26T04:25:08.597 INFO:teuthology.task.internal:roles: ubuntu@smithi055.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-26T04:25:08.597 INFO:teuthology.run_tasks:Running task console_log... 2024-04-26T04:25:08.684 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff0ce0d90d0>, signals=[15]) 2024-04-26T04:25:08.684 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-26T04:25:08.690 INFO:teuthology.task.internal:Opening connections... 2024-04-26T04:25:08.691 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi038.front.sepia.ceph.com 2024-04-26T04:25:08.692 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:25:08.775 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi055.front.sepia.ceph.com 2024-04-26T04:25:08.776 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:25:08.857 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-26T04:25:08.864 DEBUG:teuthology.orchestra.run.smithi038:> uname -m 2024-04-26T04:25:08.887 INFO:teuthology.orchestra.run.smithi038.stdout:x86_64 2024-04-26T04:25:08.888 DEBUG:teuthology.orchestra.run.smithi038:> cat /etc/os-release 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:NAME="CentOS Stream" 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:VERSION="9" 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:ID="centos" 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:VERSION_ID="9" 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:25:08.946 INFO:teuthology.orchestra.run.smithi038.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:ANSI_COLOR="0;31" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:25:08.947 INFO:teuthology.orchestra.run.smithi038.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:25:08.948 INFO:teuthology.lock.ops:Updating smithi038.front.sepia.ceph.com on lock server 2024-04-26T04:25:08.967 DEBUG:teuthology.orchestra.run.smithi055:> uname -m 2024-04-26T04:25:08.986 INFO:teuthology.orchestra.run.smithi055.stdout:x86_64 2024-04-26T04:25:08.987 DEBUG:teuthology.orchestra.run.smithi055:> cat /etc/os-release 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:NAME="CentOS Stream" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION="9" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:ID="centos" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:ID_LIKE="rhel fedora" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION_ID="9" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:PLATFORM_ID="platform:el9" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:ANSI_COLOR="0;31" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:LOGO="fedora-logo-icon" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-26T04:25:09.045 INFO:teuthology.orchestra.run.smithi055.stdout:HOME_URL="https://centos.org/" 2024-04-26T04:25:09.046 INFO:teuthology.orchestra.run.smithi055.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-26T04:25:09.046 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-26T04:25:09.046 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-26T04:25:09.046 INFO:teuthology.lock.ops:Updating smithi055.front.sepia.ceph.com on lock server 2024-04-26T04:25:09.065 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-26T04:25:09.074 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-26T04:25:09.093 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-26T04:25:09.094 DEBUG:teuthology.orchestra.run.smithi038:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:25:09.097 DEBUG:teuthology.orchestra.run.smithi055:> test '!' -e /home/ubuntu/cephtest 2024-04-26T04:25:09.113 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-26T04:25:09.120 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-26T04:25:09.120 DEBUG:teuthology.orchestra.run.smithi038:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:25:09.154 DEBUG:teuthology.orchestra.run.smithi055:> test -z $(ls -A /var/lib/ceph) 2024-04-26T04:25:09.180 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-26T04:25:09.233 INFO:teuthology.run_tasks:Running task kernel... 2024-04-26T04:25:09.247 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-26T04:25:09.248 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-26T04:25:09.248 DEBUG:teuthology.orchestra.run.smithi038:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:25:09.248 DEBUG:teuthology.orchestra.run.smithi055:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-26T04:25:09.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:25:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:25:09.266 DEBUG:teuthology.orchestra.run.smithi038:> uname -r 2024-04-26T04:25:09.266 DEBUG:teuthology.orchestra.run.smithi055:> uname -r 2024-04-26T04:25:09.326 INFO:teuthology.orchestra.run.smithi038.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:25:09.326 INFO:teuthology.orchestra.run.smithi055.stdout:5.14.0-437.el9.x86_64 2024-04-26T04:25:09.326 INFO:teuthology.task.kernel:Running kernel on smithi038: 5.14.0-437.el9.x86_64 2024-04-26T04:25:09.326 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum install -y kernel 2024-04-26T04:25:09.327 INFO:teuthology.task.kernel:Running kernel on smithi055: 5.14.0-437.el9.x86_64 2024-04-26T04:25:09.327 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum install -y kernel 2024-04-26T04:25:11.783 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - BaseOS 9.3 MB/s | 8.0 MB 00:00 2024-04-26T04:25:11.956 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - BaseOS 7.7 MB/s | 8.0 MB 00:01 2024-04-26T04:25:13.983 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-26T04:25:14.224 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - AppStream 16 MB/s | 19 MB 00:01 2024-04-26T04:25:18.867 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - CRB 5.1 MB/s | 6.2 MB 00:01 2024-04-26T04:25:19.008 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - CRB 5.8 MB/s | 6.2 MB 00:01 2024-04-26T04:25:20.708 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 16 kB 00:00 2024-04-26T04:25:21.282 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - Extras packages 13 kB/s | 16 kB 00:01 2024-04-26T04:25:21.869 INFO:teuthology.orchestra.run.smithi038.stdout:Extra Packages for Enterprise Linux 20 MB/s | 21 MB 00:01 2024-04-26T04:25:24.017 INFO:teuthology.orchestra.run.smithi055.stdout:Extra Packages for Enterprise Linux 8.0 MB/s | 21 MB 00:02 2024-04-26T04:25:27.067 INFO:teuthology.orchestra.run.smithi038.stdout:lab-extras 43 kB/s | 1.7 kB 00:00 2024-04-26T04:25:29.006 INFO:teuthology.orchestra.run.smithi038.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:25:29.006 INFO:teuthology.orchestra.run.smithi038.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:25:29.097 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:25:29.098 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:25:29.098 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:25:29.208 DEBUG:teuthology.orchestra.run.smithi038:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:25:29.350 INFO:teuthology.orchestra.run.smithi055.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-04-26T04:25:29.728 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:02 ago on Fri 26 Apr 2024 04:25:27 AM UTC. 2024-04-26T04:25:29.852 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:25:29.861 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:25:29.861 INFO:teuthology.orchestra.run.smithi038.stdout: Package Architecture Version Repository Size 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout:Reinstalling: 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 255 k 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 0 2024-04-26T04:25:29.862 INFO:teuthology.orchestra.run.smithi038.stderr:Operation aborted. 2024-04-26T04:25:29.921 INFO:teuthology.orchestra.run.smithi038.stdout:Is this ok [y/N]: 2024-04-26T04:25:29.921 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum reinstall -y kernel || true 2024-04-26T04:25:30.439 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:25:27 AM UTC. 2024-04-26T04:25:30.559 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:25:30.560 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:25:30.560 INFO:teuthology.orchestra.run.smithi038.stdout: Package Architecture Version Repository Size 2024-04-26T04:25:30.560 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:25:30.560 INFO:teuthology.orchestra.run.smithi038.stdout:Reinstalling: 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 255 k 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 0 2024-04-26T04:25:30.561 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:25:30.943 INFO:teuthology.orchestra.run.smithi038.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.0 MB/s | 255 kB 00:00 2024-04-26T04:25:30.944 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:25:30.944 INFO:teuthology.orchestra.run.smithi038.stdout:Total 665 kB/s | 255 kB 00:00 2024-04-26T04:25:30.965 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:25:31.125 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:25:31.125 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:25:31.130 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:25:31.131 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:25:31.251 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:25:31.323 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-26T04:25:31.323 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-26T04:25:31.353 INFO:teuthology.orchestra.run.smithi038.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:25:31.353 INFO:teuthology.orchestra.run.smithi038.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:25:31.420 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:25:31.421 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:25:31.421 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:25:31.541 DEBUG:teuthology.orchestra.run.smithi055:> echo no | sudo yum reinstall kernel || true 2024-04-26T04:25:32.062 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:25:29 AM UTC. 2024-04-26T04:25:32.125 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:25:32.126 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:25:32.190 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:25:32.195 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:25:32.195 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout:Reinstalling: 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 255 k 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 0 2024-04-26T04:25:32.196 INFO:teuthology.orchestra.run.smithi055.stderr:Operation aborted. 2024-04-26T04:25:32.255 INFO:teuthology.orchestra.run.smithi055.stdout:Is this ok [y/N]: 2024-04-26T04:25:32.255 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum reinstall -y kernel || true 2024-04-26T04:25:32.769 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:03 ago on Fri 26 Apr 2024 04:25:29 AM UTC. 2024-04-26T04:25:32.798 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:25:32.798 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:25:32.798 INFO:teuthology.orchestra.run.smithi038.stdout:Reinstalled: 2024-04-26T04:25:32.798 INFO:teuthology.orchestra.run.smithi038.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:25:32.799 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:25:32.799 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:25:32.890 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:25:32.890 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:25:32.890 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2024-04-26T04:25:32.890 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:25:32.890 INFO:teuthology.orchestra.run.smithi055.stdout:Reinstalling: 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 255 k 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 0 2024-04-26T04:25:32.891 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:25:32.956 DEBUG:teuthology.orchestra.run.smithi038:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:25:33.004 INFO:teuthology.orchestra.run.smithi038.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:25:33.005 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:25:33.005 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:25:33.005 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-26T04:25:33.005 DEBUG:teuthology.orchestra.run.smithi038:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:25:33.037 INFO:teuthology.orchestra.run.smithi038.stdout:ttyS1 2024-04-26T04:25:33.066 DEBUG:teuthology.parallel:result is None 2024-04-26T04:25:33.177 INFO:teuthology.orchestra.run.smithi055.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.6 MB/s | 255 kB 00:00 2024-04-26T04:25:33.177 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:25:33.178 INFO:teuthology.orchestra.run.smithi055.stdout:Total 889 kB/s | 255 kB 00:00 2024-04-26T04:25:33.199 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:25:33.380 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:25:33.381 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:25:33.387 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:25:33.387 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:25:33.523 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:25:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:25:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:25:34.271 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:25:34.271 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-26T04:25:34.952 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-26T04:25:34.952 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:25:34.953 INFO:teuthology.orchestra.run.smithi055.stdout:Reinstalled: 2024-04-26T04:25:34.953 INFO:teuthology.orchestra.run.smithi055.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-26T04:25:34.953 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:25:34.953 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:25:35.106 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q kernel | sort -rV | head -n 1 2024-04-26T04:25:35.154 INFO:teuthology.orchestra.run.smithi055.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-26T04:25:35.154 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-26T04:25:35.155 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-26T04:25:35.155 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-26T04:25:35.155 DEBUG:teuthology.orchestra.run.smithi055:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-26T04:25:35.184 INFO:teuthology.orchestra.run.smithi055.stdout:ttyS1 2024-04-26T04:25:35.207 DEBUG:teuthology.parallel:result is None 2024-04-26T04:25:35.208 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-26T04:25:35.215 INFO:teuthology.task.internal:Creating test directory... 2024-04-26T04:25:35.215 DEBUG:teuthology.orchestra.run.smithi038:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:25:35.218 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-26T04:25:35.263 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-26T04:25:35.270 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-26T04:25:35.291 INFO:teuthology.task.internal:Creating archive directory... 2024-04-26T04:25:35.292 DEBUG:teuthology.orchestra.run.smithi038:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:25:35.295 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-26T04:25:35.341 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-26T04:25:35.348 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-26T04:25:35.348 DEBUG:teuthology.orchestra.run.smithi038:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:25:35.365 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-26T04:25:35.396 INFO:teuthology.orchestra.run.smithi038.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:25:35.407 INFO:teuthology.orchestra.run.smithi038.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:25:35.409 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:25:35.420 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-26T04:25:35.420 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-26T04:25:35.428 INFO:teuthology.task.internal:Configuring sudo... 2024-04-26T04:25:35.428 DEBUG:teuthology.orchestra.run.smithi038:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:25:35.452 DEBUG:teuthology.orchestra.run.smithi055:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-26T04:25:35.484 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-26T04:25:35.492 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-26T04:25:35.492 DEBUG:teuthology.orchestra.run.smithi038:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:25:35.519 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-26T04:25:35.539 DEBUG:teuthology.orchestra.run.smithi038:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:25:35.617 DEBUG:teuthology.orchestra.run.smithi038:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:25:35.662 DEBUG:teuthology.orchestra.run.smithi038:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:25:35.747 DEBUG:teuthology.orchestra.run.smithi038:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:25:35.786 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:25:35.786 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:25:35.852 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:25:35.894 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-26T04:25:35.943 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:25:36.026 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-26T04:25:36.065 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:25:36.065 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-26T04:25:36.130 DEBUG:teuthology.orchestra.run.smithi038:> sudo service rsyslog restart 2024-04-26T04:25:36.132 DEBUG:teuthology.orchestra.run.smithi055:> sudo service rsyslog restart 2024-04-26T04:25:36.172 INFO:teuthology.orchestra.run.smithi038.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:25:36.205 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T04:25:36.640 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-26T04:25:36.648 INFO:teuthology.task.internal:Starting timer... 2024-04-26T04:25:36.648 INFO:teuthology.run_tasks:Running task pcp... 2024-04-26T04:25:36.687 INFO:teuthology.run_tasks:Running task selinux... 2024-04-26T04:25:36.702 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-26T04:25:36.703 DEBUG:teuthology.orchestra.run.smithi038:> sudo service auditd rotate 2024-04-26T04:25:36.787 INFO:teuthology.orchestra.run.smithi038.stdout:Rotating logs: 2024-04-26T04:25:36.789 DEBUG:teuthology.orchestra.run.smithi055:> sudo service auditd rotate 2024-04-26T04:25:36.846 INFO:teuthology.orchestra.run.smithi055.stdout:Rotating logs: 2024-04-26T04:25:36.848 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-26T04:25:36.848 DEBUG:teuthology.orchestra.run.smithi038:> /usr/sbin/getenforce 2024-04-26T04:25:36.870 INFO:teuthology.orchestra.run.smithi038.stdout:Permissive 2024-04-26T04:25:36.870 DEBUG:teuthology.orchestra.run.smithi055:> /usr/sbin/getenforce 2024-04-26T04:25:36.895 INFO:teuthology.orchestra.run.smithi055.stdout:Permissive 2024-04-26T04:25:36.895 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi038.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi055.front.sepia.ceph.com': 'permissive'} 2024-04-26T04:25:36.895 DEBUG:teuthology.orchestra.run.smithi038:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:25:36.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:25:36.935 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T04:25:36.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T04:25:36.961 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-26T04:25:36.961 DEBUG:teuthology.orchestra.run.smithi038:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:25:37.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo /usr/sbin/setenforce permissive 2024-04-26T04:25:37.026 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-26T04:25:37.034 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-26T04:25:37.040 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-26T04:25:37.171 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-26T04:25:37.187 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-26T04:25:37.189 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi038.front.sepia.ceph.com,smithi055.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-26T04:29:16.450 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi038.front.sepia.ceph.com'), Remote(name='ubuntu@smithi055.front.sepia.ceph.com')] 2024-04-26T04:29:16.451 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi038.front.sepia.ceph.com' 2024-04-26T04:29:16.452 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi038.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:29:16.531 DEBUG:teuthology.orchestra.run.smithi038:> true 2024-04-26T04:29:16.614 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi038.front.sepia.ceph.com' 2024-04-26T04:29:16.614 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-04-26T04:29:16.614 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-26T04:29:16.695 DEBUG:teuthology.orchestra.run.smithi055:> true 2024-04-26T04:29:16.784 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi055.front.sepia.ceph.com' 2024-04-26T04:29:16.784 INFO:teuthology.run_tasks:Running task clock... 2024-04-26T04:29:16.795 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-26T04:29:16.796 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:29:16.797 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:29:16.800 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-26T04:29:16.800 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T04:29:16.827 INFO:teuthology.orchestra.run.smithi038.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:29:16.842 INFO:teuthology.orchestra.run.smithi038.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:29:16.858 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-26T04:29:16.866 INFO:teuthology.orchestra.run.smithi038.stderr:sudo: ntpd: command not found 2024-04-26T04:29:16.873 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-26T04:29:16.877 INFO:teuthology.orchestra.run.smithi038.stdout:506 Cannot talk to daemon 2024-04-26T04:29:16.891 INFO:teuthology.orchestra.run.smithi038.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:29:16.898 INFO:teuthology.orchestra.run.smithi055.stderr:sudo: ntpd: command not found 2024-04-26T04:29:16.905 INFO:teuthology.orchestra.run.smithi038.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:29:16.910 INFO:teuthology.orchestra.run.smithi055.stdout:506 Cannot talk to daemon 2024-04-26T04:29:16.925 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-26T04:29:16.939 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-26T04:29:16.972 INFO:teuthology.orchestra.run.smithi038.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:29:16.975 INFO:teuthology.orchestra.run.smithi038.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:29:16.975 INFO:teuthology.orchestra.run.smithi038.stdout:=============================================================================== 2024-04-26T04:29:16.976 INFO:teuthology.orchestra.run.smithi038.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:16.976 INFO:teuthology.orchestra.run.smithi038.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:16.976 INFO:teuthology.orchestra.run.smithi038.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:16.976 INFO:teuthology.orchestra.run.smithi038.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:17.001 INFO:teuthology.orchestra.run.smithi055.stderr:bash: line 1: ntpq: command not found 2024-04-26T04:29:17.002 INFO:teuthology.orchestra.run.smithi055.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T04:29:17.002 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================== 2024-04-26T04:29:17.002 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:17.003 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:17.003 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:17.003 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T04:29:17.003 INFO:teuthology.run_tasks:Running task install... 2024-04-26T04:29:17.012 DEBUG:teuthology.task.install:project ceph 2024-04-26T04:29:17.012 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'}} 2024-04-26T04:29:17.012 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:29:17.012 INFO:teuthology.task.install:Using flavor: default 2024-04-26T04:29:17.020 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-26T04:29:17.020 INFO:teuthology.task.install:extra packages: [] 2024-04-26T04:29:17.021 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:29:17.021 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:29:17.021 INFO:teuthology.packaging:ref: None 2024-04-26T04:29:17.021 INFO:teuthology.packaging:tag: None 2024-04-26T04:29:17.021 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:29:17.021 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:29:17.022 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:29:17.024 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a', 'tag': None, 'wait_for_package': False} 2024-04-26T04:29:17.025 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:29:17.025 INFO:teuthology.packaging:ref: None 2024-04-26T04:29:17.025 INFO:teuthology.packaging:tag: None 2024-04-26T04:29:17.025 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:29:17.029 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:29:17.029 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:29:17.275 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:29:17.275 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:29:17.344 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/ 2024-04-26T04:29:17.345 INFO:teuthology.task.install.rpm:Package version is 17.2.7-904.gd8c6b0a3 2024-04-26T04:29:17.395 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:29:17.395 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:29:17.395 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:29:17.422 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:29:17.422 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:29:17.422 INFO:teuthology.packaging:ref: None 2024-04-26T04:29:17.422 INFO:teuthology.packaging:tag: None 2024-04-26T04:29:17.422 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:29:17.422 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:29:17.423 DEBUG:teuthology.orchestra.run.smithi038:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:29:17.489 DEBUG:teuthology.orchestra.run.smithi038:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:29:17.560 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/d8c6b0a3ba8ca216f4aadb85bc7242e7c3b18d34/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-26T04:29:17.560 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:29:17.560 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-26T04:29:17.567 DEBUG:teuthology.orchestra.run.smithi038:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:29:17.591 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-26T04:29:17.591 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:29:17.591 INFO:teuthology.packaging:ref: None 2024-04-26T04:29:17.591 INFO:teuthology.packaging:tag: None 2024-04-26T04:29:17.591 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:29:17.591 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:29:17.591 DEBUG:teuthology.orchestra.run.smithi055:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-26T04:29:17.632 INFO:teuthology.orchestra.run.smithi038.stdout:check_obsoletes = 1 2024-04-26T04:29:17.633 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum clean all 2024-04-26T04:29:17.661 DEBUG:teuthology.orchestra.run.smithi055:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-26T04:29:17.739 DEBUG:teuthology.orchestra.run.smithi055:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-26T04:29:17.806 INFO:teuthology.orchestra.run.smithi055.stdout:check_obsoletes = 1 2024-04-26T04:29:17.808 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum clean all 2024-04-26T04:29:17.866 INFO:teuthology.orchestra.run.smithi038.stdout:48 files removed 2024-04-26T04:29:17.893 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-radosgw 2024-04-26T04:29:18.045 INFO:teuthology.orchestra.run.smithi055.stdout:48 files removed 2024-04-26T04:29:18.073 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-radosgw 2024-04-26T04:29:18.766 INFO:teuthology.orchestra.run.smithi055.stdout:ceph packages for x86_64 182 kB/s | 77 kB 00:00 2024-04-26T04:29:18.805 INFO:teuthology.orchestra.run.smithi038.stdout:ceph packages for x86_64 121 kB/s | 77 kB 00:00 2024-04-26T04:29:19.100 INFO:teuthology.orchestra.run.smithi055.stdout:ceph noarch packages 37 kB/s | 11 kB 00:00 2024-04-26T04:29:19.239 INFO:teuthology.orchestra.run.smithi038.stdout:ceph noarch packages 27 kB/s | 11 kB 00:00 2024-04-26T04:29:19.551 INFO:teuthology.orchestra.run.smithi055.stdout:ceph source packages 4.3 kB/s | 1.8 kB 00:00 2024-04-26T04:29:19.622 INFO:teuthology.orchestra.run.smithi038.stdout:ceph source packages 5.1 kB/s | 1.8 kB 00:00 2024-04-26T04:29:20.411 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - BaseOS 9.7 MB/s | 8.0 MB 00:00 2024-04-26T04:29:20.575 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - BaseOS 8.8 MB/s | 8.0 MB 00:00 2024-04-26T04:29:23.063 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-04-26T04:29:23.218 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-04-26T04:29:27.331 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.2 MB 00:00 2024-04-26T04:29:27.561 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.2 MB 00:00 2024-04-26T04:29:28.829 INFO:teuthology.orchestra.run.smithi055.stdout:CentOS Stream 9 - Extras packages 53 kB/s | 16 kB 00:00 2024-04-26T04:29:29.042 INFO:teuthology.orchestra.run.smithi038.stdout:CentOS Stream 9 - Extras packages 58 kB/s | 16 kB 00:00 2024-04-26T04:29:29.839 INFO:teuthology.orchestra.run.smithi055.stdout:Extra Packages for Enterprise Linux 24 MB/s | 21 MB 00:00 2024-04-26T04:29:35.031 INFO:teuthology.orchestra.run.smithi055.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-04-26T04:29:37.270 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout:Upgrading: 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:29:37.272 INFO:teuthology.orchestra.run.smithi055.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:29:37.273 INFO:teuthology.orchestra.run.smithi055.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:29:37.274 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:29:37.275 INFO:teuthology.orchestra.run.smithi055.stdout:Install 22 Packages 2024-04-26T04:29:37.275 INFO:teuthology.orchestra.run.smithi055.stdout:Upgrade 2 Packages 2024-04-26T04:29:37.275 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:29:37.275 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 58 M 2024-04-26T04:29:37.275 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:29:38.924 INFO:teuthology.orchestra.run.smithi055.stdout:(1/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9.x 9.4 MB/s | 14 MB 00:01 2024-04-26T04:29:39.240 INFO:teuthology.orchestra.run.smithi055.stdout:(2/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x8 12 MB/s | 22 MB 00:01 2024-04-26T04:29:39.265 INFO:teuthology.orchestra.run.smithi055.stdout:(3/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 74 kB/s | 25 kB 00:00 2024-04-26T04:29:39.316 INFO:teuthology.orchestra.run.smithi055.stdout:(4/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 8.3 MB/s | 637 kB 00:00 2024-04-26T04:29:39.341 INFO:teuthology.orchestra.run.smithi055.stdout:(5/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 5.9 MB/s | 457 kB 00:00 2024-04-26T04:29:39.575 INFO:teuthology.orchestra.run.smithi055.stdout:(6/24): python3-ceph-argparse-17.2.7-904.gd8c6b 193 kB/s | 45 kB 00:00 2024-04-26T04:29:39.642 INFO:teuthology.orchestra.run.smithi055.stdout:(7/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 15 MB/s | 4.9 MB 00:00 2024-04-26T04:29:39.658 INFO:teuthology.orchestra.run.smithi055.stdout:(8/24): python3-ceph-common-17.2.7-904.gd8c6b0a 1.4 MB/s | 117 kB 00:00 2024-04-26T04:29:39.709 INFO:teuthology.orchestra.run.smithi055.stdout:(9/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 2.3 MB/s | 156 kB 00:00 2024-04-26T04:29:39.842 INFO:teuthology.orchestra.run.smithi055.stdout:(10/24): python3-rados-17.2.7-904.gd8c6b0a3.el9 1.7 MB/s | 320 kB 00:00 2024-04-26T04:29:39.868 INFO:teuthology.orchestra.run.smithi055.stdout:(11/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 1.8 MB/s | 293 kB 00:00 2024-04-26T04:29:39.926 INFO:teuthology.orchestra.run.smithi055.stdout:(12/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.2 MB/s | 100 kB 00:00 2024-04-26T04:29:39.995 INFO:teuthology.orchestra.run.smithi055.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 160 kB 00:00 2024-04-26T04:29:40.020 INFO:teuthology.orchestra.run.smithi055.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 489 kB/s | 45 kB 00:00 2024-04-26T04:29:40.062 INFO:teuthology.orchestra.run.smithi055.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 9.7 MB/s | 662 kB 00:00 2024-04-26T04:29:40.088 INFO:teuthology.orchestra.run.smithi055.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.3 MB/s | 292 kB 00:00 2024-04-26T04:29:40.113 INFO:teuthology.orchestra.run.smithi055.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 849 kB/s | 42 kB 00:00 2024-04-26T04:29:40.472 INFO:teuthology.orchestra.run.smithi055.stdout:(18/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86 1.6 MB/s | 5.0 MB 00:03 2024-04-26T04:29:40.639 INFO:teuthology.orchestra.run.smithi055.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 92 kB/s | 49 kB 00:00 2024-04-26T04:29:40.772 INFO:teuthology.orchestra.run.smithi055.stdout:(20/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 451 kB/s | 309 kB 00:00 2024-04-26T04:29:40.789 INFO:teuthology.orchestra.run.smithi055.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 212 kB/s | 67 kB 00:00 2024-04-26T04:29:41.048 INFO:teuthology.orchestra.run.smithi055.stdout:(22/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 11 MB/s | 2.9 MB 00:00 2024-04-26T04:29:41.083 INFO:teuthology.orchestra.run.smithi055.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 3.6 MB/s | 1.6 MB 00:00 2024-04-26T04:29:41.357 INFO:teuthology.orchestra.run.smithi055.stdout:(24/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 5.3 MB/s | 3.1 MB 00:00 2024-04-26T04:29:41.361 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:29:41.361 INFO:teuthology.orchestra.run.smithi055.stdout:Total 14 MB/s | 58 MB 00:04 2024-04-26T04:29:41.424 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:29:41.462 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:29:41.463 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:29:41.920 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:29:41.921 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:29:43.190 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:29:43.606 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:29:43.701 INFO:teuthology.orchestra.run.smithi055.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:29:43.794 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:29:43.874 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:29:43.903 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:29:43.991 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:29:44.046 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:29:44.074 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:29:44.533 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:29:44.564 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:29:44.752 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:29:44.789 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:29:44.833 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:29:44.869 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:29:44.941 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:29:44.967 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:29:45.015 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:29:45.055 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:29:45.092 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:29:45.125 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:29:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:29:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:29:45.578 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:29:45.654 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:29:45.806 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:29:48.318 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:29:48.361 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:29:48.898 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:29:48.940 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:29:49.019 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:29:49.020 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:29:49.020 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:29:49.020 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:29:49.050 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:29:49.884 INFO:teuthology.orchestra.run.smithi038.stdout:Extra Packages for Enterprise Linux 1.0 MB/s | 21 MB 00:20 2024-04-26T04:29:55.184 INFO:teuthology.orchestra.run.smithi038.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-04-26T04:29:57.422 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:29:57.424 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:29:57.424 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-radosgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 14 M 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout:Upgrading: 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: librados2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.1 M 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: librbd1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-base x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 5.0 M 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 22 M 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-selinux x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: libcephfs2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 637 k 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-26T04:29:57.425 INFO:teuthology.orchestra.run.smithi038.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: libradosstriper1 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 457 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: librgw2 x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 4.9 M 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-ceph-argparse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 45 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-ceph-common x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 117 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cephfs x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 156 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rados x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 320 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 293 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rgw x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 100 k 2024-04-26T04:29:57.426 INFO:teuthology.orchestra.run.smithi038.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout:Install 22 Packages 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout:Upgrade 2 Packages 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 58 M 2024-04-26T04:29:57.427 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:29:58.427 INFO:teuthology.orchestra.run.smithi038.stdout:(1/24): ceph-base-17.2.7-904.gd8c6b0a3.el9.x86_ 7.1 MB/s | 5.0 MB 00:00 2024-04-26T04:29:58.486 INFO:teuthology.orchestra.run.smithi038.stdout:(2/24): ceph-selinux-17.2.7-904.gd8c6b0a3.el9.x 432 kB/s | 25 kB 00:00 2024-04-26T04:29:58.577 INFO:teuthology.orchestra.run.smithi038.stdout:(3/24): libcephfs2-17.2.7-904.gd8c6b0a3.el9.x86 6.8 MB/s | 637 kB 00:00 2024-04-26T04:29:58.653 INFO:teuthology.orchestra.run.smithi038.stdout:(4/24): libradosstriper1-17.2.7-904.gd8c6b0a3.e 5.9 MB/s | 457 kB 00:00 2024-04-26T04:29:58.921 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:29:58.921 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /sys 2024-04-26T04:29:58.921 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /proc 2024-04-26T04:29:58.921 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /mnt 2024-04-26T04:29:58.921 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /var/tmp 2024-04-26T04:29:58.922 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /home 2024-04-26T04:29:58.922 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /root 2024-04-26T04:29:58.922 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /tmp 2024-04-26T04:29:58.922 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:29:59.220 INFO:teuthology.orchestra.run.smithi038.stdout:(5/24): ceph-radosgw-17.2.7-904.gd8c6b0a3.el9.x 9.5 MB/s | 14 MB 00:01 2024-04-26T04:29:59.295 INFO:teuthology.orchestra.run.smithi038.stdout:(6/24): librgw2-17.2.7-904.gd8c6b0a3.el9.x86_64 7.6 MB/s | 4.9 MB 00:00 2024-04-26T04:29:59.312 INFO:teuthology.orchestra.run.smithi038.stdout:(7/24): python3-ceph-argparse-17.2.7-904.gd8c6b 491 kB/s | 45 kB 00:00 2024-04-26T04:29:59.337 INFO:teuthology.orchestra.run.smithi038.stdout:(8/24): python3-ceph-common-17.2.7-904.gd8c6b0a 2.7 MB/s | 117 kB 00:00 2024-04-26T04:29:59.362 INFO:teuthology.orchestra.run.smithi038.stdout:(9/24): python3-cephfs-17.2.7-904.gd8c6b0a3.el9 3.0 MB/s | 156 kB 00:00 2024-04-26T04:29:59.404 INFO:teuthology.orchestra.run.smithi038.stdout:(10/24): python3-rados-17.2.7-904.gd8c6b0a3.el9 4.7 MB/s | 320 kB 00:00 2024-04-26T04:29:59.430 INFO:teuthology.orchestra.run.smithi038.stdout:(11/24): python3-rbd-17.2.7-904.gd8c6b0a3.el9.x 4.3 MB/s | 293 kB 00:00 2024-04-26T04:29:59.463 INFO:teuthology.orchestra.run.smithi038.stdout:(12/24): python3-rgw-17.2.7-904.gd8c6b0a3.el9.x 1.7 MB/s | 100 kB 00:00 2024-04-26T04:29:59.755 INFO:teuthology.orchestra.run.smithi038.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 155 kB/s | 45 kB 00:00 2024-04-26T04:29:59.781 INFO:teuthology.orchestra.run.smithi038.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 457 kB/s | 160 kB 00:00 2024-04-26T04:29:59.831 INFO:teuthology.orchestra.run.smithi038.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 8.6 MB/s | 662 kB 00:00 2024-04-26T04:29:59.856 INFO:teuthology.orchestra.run.smithi038.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.8 MB/s | 292 kB 00:00 2024-04-26T04:29:59.882 INFO:teuthology.orchestra.run.smithi038.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 848 kB/s | 42 kB 00:00 2024-04-26T04:30:00.034 INFO:teuthology.orchestra.run.smithi038.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 319 kB/s | 49 kB 00:00 2024-04-26T04:30:00.118 INFO:teuthology.orchestra.run.smithi038.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.2 MB/s | 309 kB 00:00 2024-04-26T04:30:00.143 INFO:teuthology.orchestra.run.smithi038.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 620 kB/s | 67 kB 00:00 2024-04-26T04:30:00.224 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:30:00.303 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:30:00.303 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:00.303 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:30:00.303 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:30:00.303 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:30:00.303 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:00.305 INFO:teuthology.orchestra.run.smithi055.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:30:00.352 INFO:teuthology.orchestra.run.smithi038.stdout:(21/24): ceph-common-17.2.7-904.gd8c6b0a3.el9.x 8.6 MB/s | 22 MB 00:02 2024-04-26T04:30:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:30:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:30:00.402 INFO:teuthology.orchestra.run.smithi038.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 5.6 MB/s | 1.6 MB 00:00 2024-04-26T04:30:00.510 INFO:teuthology.orchestra.run.smithi038.stdout:(23/24): librados2-17.2.7-904.gd8c6b0a3.el9.x86 8.4 MB/s | 3.1 MB 00:00 2024-04-26T04:30:00.628 INFO:teuthology.orchestra.run.smithi038.stdout:(24/24): librbd1-17.2.7-904.gd8c6b0a3.el9.x86_6 11 MB/s | 2.9 MB 00:00 2024-04-26T04:30:00.633 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:00.633 INFO:teuthology.orchestra.run.smithi038.stdout:Total 18 MB/s | 58 MB 00:03 2024-04-26T04:30:00.696 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:30:00.734 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:30:00.734 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:30:01.189 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:30:01.190 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:30:01.826 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:30:01.827 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:30:01.828 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:30:02.451 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:30:02.569 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:30:02.569 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:02.569 INFO:teuthology.orchestra.run.smithi055.stdout:Upgraded: 2024-04-26T04:30:02.569 INFO:teuthology.orchestra.run.smithi055.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.569 INFO:teuthology.orchestra.run.smithi055.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.569 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:30:02.570 INFO:teuthology.orchestra.run.smithi055.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:02.571 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:02.850 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-test 2024-04-26T04:30:02.851 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-26T04:30:02.922 INFO:teuthology.orchestra.run.smithi038.stdout: Upgrading : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:30:02.998 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:30:03.079 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:30:03.099 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:30:03.187 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:30:03.233 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-26T04:30:03.270 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-26T04:30:03.383 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:29 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:30:03.507 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout:Install 4 Packages 2024-04-26T04:30:03.508 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:03.509 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 48 M 2024-04-26T04:30:03.509 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 192 M 2024-04-26T04:30:03.509 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:03.729 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-26T04:30:03.752 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:30:03.823 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 8/26 2024-04-26T04:30:03.860 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-26T04:30:03.904 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 10/26 2024-04-26T04:30:03.941 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:30:04.004 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:30:04.030 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:30:04.073 INFO:teuthology.orchestra.run.smithi055.stdout:(1/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 537 kB/s | 163 kB 00:00 2024-04-26T04:30:04.087 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:30:04.107 INFO:teuthology.orchestra.run.smithi055.stdout:(2/4): socat-1.7.4.1-5.el9.x86_64.rpm 906 kB/s | 305 kB 00:00 2024-04-26T04:30:04.126 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-26T04:30:04.149 INFO:teuthology.orchestra.run.smithi055.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 850 kB/s | 64 kB 00:00 2024-04-26T04:30:04.189 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-26T04:30:04.221 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-26T04:30:04.531 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-26T04:30:04.576 INFO:teuthology.orchestra.run.smithi038.stdout: Upgrading : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:30:04.667 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 18/26 2024-04-26T04:30:04.744 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 19/26 2024-04-26T04:30:04.779 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 20/26 2024-04-26T04:30:07.461 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:30:07.493 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:30:08.031 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 21/26 2024-04-26T04:30:08.072 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:30:08.153 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 22/26 2024-04-26T04:30:08.153 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-26T04:30:08.153 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-26T04:30:08.153 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:08.190 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:30:08.549 INFO:teuthology.orchestra.run.smithi055.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 10 MB/s | 48 MB 00:04 2024-04-26T04:30:08.552 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:08.552 INFO:teuthology.orchestra.run.smithi055.stdout:Total 9.6 MB/s | 48 MB 00:05 2024-04-26T04:30:08.577 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:08.597 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:08.597 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:08.978 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:08.978 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:09.515 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:09.585 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:30:09.632 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:30:09.659 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:30:14.274 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:30:14.351 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:30:16.519 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:30:16.519 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:30:16.519 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:30:16.519 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:30:16.809 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:16.810 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:17.107 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph 2024-04-26T04:30:17.648 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:43 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:17.777 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:30:17.780 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:30:17.781 INFO:teuthology.orchestra.run.smithi055.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:30:17.782 INFO:teuthology.orchestra.run.smithi055.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout:Install 33 Packages 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 27 M 2024-04-26T04:30:17.783 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 96 M 2024-04-26T04:30:17.784 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /sys 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /proc 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /mnt 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /var/tmp 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /home 2024-04-26T04:30:17.944 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /root 2024-04-26T04:30:17.945 INFO:teuthology.orchestra.run.smithi038.stdout:skipping the directory /tmp 2024-04-26T04:30:17.945 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:18.509 INFO:teuthology.orchestra.run.smithi055.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 25 kB/s | 6.5 kB 00:00 2024-04-26T04:30:18.609 INFO:teuthology.orchestra.run.smithi055.stdout:(2/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 3.9 MB/s | 1.4 MB 00:00 2024-04-26T04:30:19.259 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:30:19.343 INFO:teuthology.orchestra.run.smithi055.stdout:(3/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_6 21 MB/s | 15 MB 00:00 2024-04-26T04:30:19.355 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 24/26 2024-04-26T04:30:19.355 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:19.355 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-26T04:30:19.355 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:30:19.355 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-26T04:30:19.355 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:19.357 INFO:teuthology.orchestra.run.smithi038.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:30:19.446 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-26T04:30:19.446 INFO:teuthology.orchestra.run.smithi038.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:30:19.476 INFO:teuthology.orchestra.run.smithi055.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 1.8 MB/s | 240 kB 00:00 2024-04-26T04:30:19.602 INFO:teuthology.orchestra.run.smithi055.stdout:(5/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.0 MB/s | 253 kB 00:00 2024-04-26T04:30:19.677 INFO:teuthology.orchestra.run.smithi055.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 16 MB/s | 1.2 MB 00:00 2024-04-26T04:30:19.719 INFO:teuthology.orchestra.run.smithi055.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-04-26T04:30:19.752 INFO:teuthology.orchestra.run.smithi055.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 4.0 MB/s | 135 kB 00:00 2024-04-26T04:30:19.786 INFO:teuthology.orchestra.run.smithi055.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 3.7 MB/s | 125 kB 00:00 2024-04-26T04:30:19.820 INFO:teuthology.orchestra.run.smithi055.stdout:(10/33): python3-urllib3-1.26.5-5.el9.noarch.rp 6.3 MB/s | 215 kB 00:00 2024-04-26T04:30:19.853 INFO:teuthology.orchestra.run.smithi055.stdout:(11/33): python3-mako-1.1.4-6.el9.noarch.rpm 5.0 MB/s | 172 kB 00:00 2024-04-26T04:30:19.904 INFO:teuthology.orchestra.run.smithi055.stdout:(12/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_ 1.3 MB/s | 2.1 MB 00:01 2024-04-26T04:30:19.921 INFO:teuthology.orchestra.run.smithi055.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 519 kB/s | 35 kB 00:00 2024-04-26T04:30:19.946 INFO:teuthology.orchestra.run.smithi055.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.2 MB/s | 51 kB 00:00 2024-04-26T04:30:20.080 INFO:teuthology.orchestra.run.smithi055.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 263 kB/s | 42 kB 00:00 2024-04-26T04:30:20.246 INFO:teuthology.orchestra.run.smithi055.stdout:(16/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_ 2.3 MB/s | 3.9 MB 00:01 2024-04-26T04:30:20.380 INFO:teuthology.orchestra.run.smithi055.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 100 kB/s | 43 kB 00:00 2024-04-26T04:30:20.472 INFO:teuthology.orchestra.run.smithi055.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 117 kB/s | 11 kB 00:00 2024-04-26T04:30:20.556 INFO:teuthology.orchestra.run.smithi055.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 361 kB/s | 172 kB 00:00 2024-04-26T04:30:20.589 INFO:teuthology.orchestra.run.smithi055.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 151 kB/s | 18 kB 00:00 2024-04-26T04:30:20.648 INFO:teuthology.orchestra.run.smithi055.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 252 kB/s | 23 kB 00:00 2024-04-26T04:30:20.702 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:30:20.703 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/26 2024-04-26T04:30:20.703 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/26 2024-04-26T04:30:20.703 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/26 2024-04-26T04:30:20.704 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/26 2024-04-26T04:30:20.704 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/26 2024-04-26T04:30:20.704 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 6/26 2024-04-26T04:30:20.704 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 7/26 2024-04-26T04:30:20.704 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x 8/26 2024-04-26T04:30:20.705 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86 9/26 2024-04-26T04:30:20.705 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 10/26 2024-04-26T04:30:20.705 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 11/26 2024-04-26T04:30:20.705 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 12/26 2024-04-26T04:30:20.705 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 13/26 2024-04-26T04:30:20.705 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-26T04:30:20.706 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-26T04:30:20.706 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-26T04:30:20.706 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-26T04:30:20.707 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-26T04:30:20.707 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-26T04:30:20.707 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-26T04:30:20.707 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-26T04:30:20.707 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-26T04:30:20.708 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 23/26 2024-04-26T04:30:20.708 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-26T04:30:20.708 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 25/26 2024-04-26T04:30:20.708 INFO:teuthology.orchestra.run.smithi055.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 179 kB/s | 19 kB 00:00 2024-04-26T04:30:20.748 INFO:teuthology.orchestra.run.smithi055.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 196 kB/s | 20 kB 00:00 2024-04-26T04:30:20.774 INFO:teuthology.orchestra.run.smithi055.stdout:(24/33): python3-cherrypy-18.6.1-2.el9.noarch.r 680 kB/s | 358 kB 00:00 2024-04-26T04:30:20.832 INFO:teuthology.orchestra.run.smithi055.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 346 kB/s | 46 kB 00:00 2024-04-26T04:30:20.866 INFO:teuthology.orchestra.run.smithi055.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 672 kB/s | 79 kB 00:00 2024-04-26T04:30:20.941 INFO:teuthology.orchestra.run.smithi055.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 151 kB/s | 16 kB 00:00 2024-04-26T04:30:20.975 INFO:teuthology.orchestra.run.smithi055.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.3 MB/s | 272 kB 00:00 2024-04-26T04:30:21.000 INFO:teuthology.orchestra.run.smithi055.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 672 kB/s | 90 kB 00:00 2024-04-26T04:30:21.059 INFO:teuthology.orchestra.run.smithi055.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 305 kB/s | 36 kB 00:00 2024-04-26T04:30:21.101 INFO:teuthology.orchestra.run.smithi055.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.8 MB/s | 230 kB 00:00 2024-04-26T04:30:21.209 INFO:teuthology.orchestra.run.smithi055.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 133 kB/s | 20 kB 00:00 2024-04-26T04:30:21.242 INFO:teuthology.orchestra.run.smithi055.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 1.7 MB/s | 427 kB 00:00 2024-04-26T04:30:21.246 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:21.246 INFO:teuthology.orchestra.run.smithi055.stdout:Total 7.8 MB/s | 27 MB 00:03 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout:Upgraded: 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout: librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout: librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:30:21.345 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-radosgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-selinux-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: libradosstriper1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: librgw2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-26T04:30:21.346 INFO:teuthology.orchestra.run.smithi038.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-ceph-argparse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-ceph-common-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:21.347 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:21.366 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:21.407 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:21.407 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:21.590 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-test 2024-04-26T04:30:21.691 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:21.693 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:22.131 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:27 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:22.253 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-test x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 48 M 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout: libcephsqlite x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 163 k 2024-04-26T04:30:22.254 INFO:teuthology.orchestra.run.smithi038.stdout: socat x86_64 1.7.4.1-5.el9 appstream 305 k 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout:Install 4 Packages 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 48 M 2024-04-26T04:30:22.255 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 192 M 2024-04-26T04:30:22.256 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:30:22.520 INFO:teuthology.orchestra.run.smithi038.stdout:(1/4): socat-1.7.4.1-5.el9.x86_64.rpm 1.6 MB/s | 305 kB 00:00 2024-04-26T04:30:22.562 INFO:teuthology.orchestra.run.smithi038.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-04-26T04:30:22.629 INFO:teuthology.orchestra.run.smithi038.stdout:(3/4): libcephsqlite-17.2.7-904.gd8c6b0a3.el9.x 553 kB/s | 163 kB 00:00 2024-04-26T04:30:22.887 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:22.945 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:30:23.004 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:30:23.045 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:30:23.142 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:30:23.343 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:30:23.390 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:30:23.424 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:30:23.454 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:30:23.488 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:30:23.522 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:30:23.563 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:30:23.606 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:30:23.640 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:30:23.679 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:30:23.756 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:30:23.909 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:30:23.990 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:30:24.175 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:30:24.212 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:30:24.271 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:30:24.334 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:30:24.533 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:30:24.583 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:30:24.664 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:30:24.718 INFO:teuthology.orchestra.run.smithi038.stdout:(4/4): ceph-test-17.2.7-904.gd8c6b0a3.el9.x86_6 20 MB/s | 48 MB 00:02 2024-04-26T04:30:24.721 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:24.721 INFO:teuthology.orchestra.run.smithi038.stdout:Total 20 MB/s | 48 MB 00:02 2024-04-26T04:30:24.746 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:30:24.762 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:30:24.767 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:30:24.767 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:30:24.798 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:30:24.921 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:30:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:30:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:30:25.148 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:30:25.149 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:30:25.150 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:30:25.150 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:25.150 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:30:25.150 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:30:25.150 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:30:25.151 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:25.680 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:30:25.761 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-04-26T04:30:25.801 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/4 2024-04-26T04:30:25.828 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:30:26.542 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:30:26.570 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:30:26.570 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:26.570 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:30:26.570 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:30:26.570 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:30:26.570 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:30:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:30:26.984 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:26.984 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:30:26.984 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:30:26.984 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:30:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:27.202 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:30:27.225 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:30:27.226 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:27.226 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:30:27.226 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:30:27.226 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:30:27.226 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:27.283 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:30:28.956 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:30:28.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:30:28.958 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:30:28.958 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:30:28.958 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:30:28.958 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:30:28.958 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:30:28.958 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:30:28.959 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:30:28.959 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:30:28.959 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:30:28.959 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:30:28.959 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:30:29.814 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:30:29.814 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:29.814 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:29.815 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:29.815 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:29.815 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:29.815 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:30:29.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:30:29.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:29.819 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:30.141 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-base 2024-04-26T04:30:30.418 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/4 2024-04-26T04:30:30.494 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:30:30.672 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:56 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:30.753 INFO:teuthology.orchestra.run.smithi055.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:30:30.796 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:30.797 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:30:30.797 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:31.044 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install cephadm 2024-04-26T04:30:31.567 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:57 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:31.693 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:31.694 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:30:31.695 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:31.695 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 82 k 2024-04-26T04:30:31.695 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 372 k 2024-04-26T04:30:31.695 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:32.034 INFO:teuthology.orchestra.run.smithi055.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 242 kB/s | 82 kB 00:00 2024-04-26T04:30:32.034 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:32.034 INFO:teuthology.orchestra.run.smithi055.stdout:Total 241 kB/s | 82 kB 00:00 2024-04-26T04:30:32.034 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:32.040 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:32.040 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:32.042 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:32.042 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:32.116 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:32.561 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:32.664 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:32.797 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/4 2024-04-26T04:30:32.797 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/4 2024-04-26T04:30:32.797 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/4 2024-04-26T04:30:32.797 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 3/4 2024-04-26T04:30:32.994 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:33.051 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-04-26T04:30:33.051 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-test-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout: libcephsqlite-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:33.052 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:33.231 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:33.232 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:33.232 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:33.232 INFO:teuthology.orchestra.run.smithi055.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:33.232 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:33.232 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:33.319 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph 2024-04-26T04:30:33.421 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:30:33.850 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:38 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:33.953 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:59 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:33.976 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:33.978 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:33.978 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: ceph x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 6.5 k 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mds x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.1 M 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 1.4 M 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-modules-core noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 240 k 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mon x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 3.9 M 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-osd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 15 M 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-26T04:30:33.979 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-04-26T04:30:33.980 INFO:teuthology.orchestra.run.smithi038.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-urllib3 noarch 1.26.5-5.el9 baseos 215 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout:Install 33 Packages 2024-04-26T04:30:33.981 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:33.982 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 27 M 2024-04-26T04:30:33.982 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 96 M 2024-04-26T04:30:33.982 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:30:34.077 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:30:34.078 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:34.079 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 137 k 2024-04-26T04:30:34.079 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 379 k 2024-04-26T04:30:34.079 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:34.448 INFO:teuthology.orchestra.run.smithi055.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 372 kB/s | 137 kB 00:00 2024-04-26T04:30:34.449 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:34.449 INFO:teuthology.orchestra.run.smithi055.stdout:Total 370 kB/s | 137 kB 00:00 2024-04-26T04:30:34.450 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:34.458 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:34.458 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:34.517 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:34.517 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:34.670 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:30:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:30:34.775 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:34.775 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:30:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:34.929 INFO:teuthology.orchestra.run.smithi038.stdout:(1/33): ceph-17.2.7-904.gd8c6b0a3.el9.x86_64.rp 30 kB/s | 6.5 kB 00:00 2024-04-26T04:30:35.062 INFO:teuthology.orchestra.run.smithi038.stdout:(2/33): ceph-mgr-17.2.7-904.gd8c6b0a3.el9.x86_6 4.1 MB/s | 1.4 MB 00:00 2024-04-26T04:30:35.513 INFO:teuthology.orchestra.run.smithi038.stdout:(3/33): ceph-mds-17.2.7-904.gd8c6b0a3.el9.x86_6 2.6 MB/s | 2.1 MB 00:00 2024-04-26T04:30:35.613 INFO:teuthology.orchestra.run.smithi038.stdout:(4/33): ceph-mgr-modules-core-17.2.7-904.gd8c6b 2.4 MB/s | 240 kB 00:00 2024-04-26T04:30:35.623 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:30:35.623 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:35.623 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:35.623 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:35.623 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:35.623 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:35.818 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr 2024-04-26T04:30:35.955 INFO:teuthology.orchestra.run.smithi038.stdout:(5/33): ceph-osd-17.2.7-904.gd8c6b0a3.el9.x86_6 17 MB/s | 15 MB 00:00 2024-04-26T04:30:36.089 INFO:teuthology.orchestra.run.smithi038.stdout:(6/33): ceph-mon-17.2.7-904.gd8c6b0a3.el9.x86_6 3.4 MB/s | 3.9 MB 00:01 2024-04-26T04:30:36.348 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:02 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:36.430 INFO:teuthology.orchestra.run.smithi055.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:30:36.472 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:36.474 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:30:36.474 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:36.534 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:30:36.765 INFO:teuthology.orchestra.run.smithi038.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 220 kB/s | 253 kB 00:01 2024-04-26T04:30:36.956 INFO:teuthology.orchestra.run.smithi038.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 123 kB/s | 106 kB 00:00 2024-04-26T04:30:36.998 INFO:teuthology.orchestra.run.smithi038.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 1.2 MB/s | 1.2 MB 00:01 2024-04-26T04:30:37.024 INFO:teuthology.orchestra.run.smithi038.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 523 kB/s | 135 kB 00:00 2024-04-26T04:30:37.067 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:03 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:37.124 INFO:teuthology.orchestra.run.smithi038.stdout:(11/33): python3-requests-2.25.1-8.el9.noarch.r 748 kB/s | 125 kB 00:00 2024-04-26T04:30:37.174 INFO:teuthology.orchestra.run.smithi038.stdout:(12/33): python3-urllib3-1.26.5-5.el9.noarch.rp 1.2 MB/s | 215 kB 00:00 2024-04-26T04:30:37.193 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:30:37.194 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout:Installing weak dependencies: 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout:Install 7 Packages 2024-04-26T04:30:37.195 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:37.196 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 3.7 M 2024-04-26T04:30:37.196 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 80 M 2024-04-26T04:30:37.196 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:37.200 INFO:teuthology.orchestra.run.smithi038.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 979 kB/s | 172 kB 00:00 2024-04-26T04:30:37.275 INFO:teuthology.orchestra.run.smithi038.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 230 kB/s | 35 kB 00:00 2024-04-26T04:30:37.334 INFO:teuthology.orchestra.run.smithi038.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 322 kB/s | 51 kB 00:00 2024-04-26T04:30:37.359 INFO:teuthology.orchestra.run.smithi038.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 262 kB/s | 42 kB 00:00 2024-04-26T04:30:37.442 INFO:teuthology.orchestra.run.smithi038.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 260 kB/s | 43 kB 00:00 2024-04-26T04:30:37.501 INFO:teuthology.orchestra.run.smithi038.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 183 kB/s | 11 kB 00:00 2024-04-26T04:30:37.543 INFO:teuthology.orchestra.run.smithi038.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 822 kB/s | 172 kB 00:00 2024-04-26T04:30:37.568 INFO:teuthology.orchestra.run.smithi038.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 264 kB/s | 18 kB 00:00 2024-04-26T04:30:37.602 INFO:teuthology.orchestra.run.smithi038.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 397 kB/s | 23 kB 00:00 2024-04-26T04:30:37.627 INFO:teuthology.orchestra.run.smithi038.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-04-26T04:30:37.644 INFO:teuthology.orchestra.run.smithi038.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 257 kB/s | 19 kB 00:00 2024-04-26T04:30:37.653 INFO:teuthology.orchestra.run.smithi055.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 40 kB/s | 15 kB 00:00 2024-04-26T04:30:37.669 INFO:teuthology.orchestra.run.smithi038.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 292 kB/s | 20 kB 00:00 2024-04-26T04:30:37.686 INFO:teuthology.orchestra.run.smithi038.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 786 kB/s | 46 kB 00:00 2024-04-26T04:30:37.694 INFO:teuthology.orchestra.run.smithi055.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 60 kB/s | 24 kB 00:00 2024-04-26T04:30:37.712 INFO:teuthology.orchestra.run.smithi038.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.1 MB/s | 79 kB 00:00 2024-04-26T04:30:37.737 INFO:teuthology.orchestra.run.smithi038.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 326 kB/s | 16 kB 00:00 2024-04-26T04:30:37.762 INFO:teuthology.orchestra.run.smithi038.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-04-26T04:30:37.770 INFO:teuthology.orchestra.run.smithi055.stdout:(3/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 6.9 MB/s | 3.4 MB 00:00 2024-04-26T04:30:37.788 INFO:teuthology.orchestra.run.smithi038.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-04-26T04:30:37.804 INFO:teuthology.orchestra.run.smithi038.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 532 kB/s | 36 kB 00:00 2024-04-26T04:30:37.830 INFO:teuthology.orchestra.run.smithi038.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 3.3 MB/s | 230 kB 00:00 2024-04-26T04:30:37.863 INFO:teuthology.orchestra.run.smithi038.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 341 kB/s | 20 kB 00:00 2024-04-26T04:30:37.914 INFO:teuthology.orchestra.run.smithi038.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.3 MB/s | 427 kB 00:00 2024-04-26T04:30:37.917 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:37.918 INFO:teuthology.orchestra.run.smithi038.stdout:Total 6.9 MB/s | 27 MB 00:03 2024-04-26T04:30:37.970 INFO:teuthology.orchestra.run.smithi055.stdout:(4/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 28 kB/s | 9.0 kB 00:00 2024-04-26T04:30:38.038 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:30:38.071 INFO:teuthology.orchestra.run.smithi055.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 102 kB/s | 31 kB 00:00 2024-04-26T04:30:38.079 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:30:38.079 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:30:38.162 INFO:teuthology.orchestra.run.smithi055.stdout:(6/7): python3-jwt-2.4.0-1.el9.noarch.rpm 87 kB/s | 41 kB 00:00 2024-04-26T04:30:38.363 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:30:38.363 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:30:38.537 INFO:teuthology.orchestra.run.smithi055.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 332 kB/s | 188 kB 00:00 2024-04-26T04:30:38.540 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:38.540 INFO:teuthology.orchestra.run.smithi055.stdout:Total 2.7 MB/s | 3.7 MB 00:01 2024-04-26T04:30:38.540 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:38.551 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:38.551 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:38.622 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:38.624 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:38.931 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:38.989 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:30:39.024 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:30:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:30:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:30:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:30:39.521 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:30:39.583 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-26T04:30:39.616 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-26T04:30:39.649 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-26T04:30:39.746 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-26T04:30:39.831 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-26T04:30:39.877 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-26T04:30:39.912 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-26T04:30:39.942 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-04-26T04:30:39.968 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-04-26T04:30:40.001 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-04-26T04:30:40.042 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-04-26T04:30:40.086 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-04-26T04:30:40.119 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-04-26T04:30:40.158 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-04-26T04:30:40.204 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:30:40.236 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-04-26T04:30:40.250 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:30:40.395 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-04-26T04:30:40.478 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-04-26T04:30:40.547 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 18/33 2024-04-26T04:30:40.592 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-04-26T04:30:40.651 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-04-26T04:30:40.713 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-04-26T04:30:40.911 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-04-26T04:30:40.971 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-04-26T04:30:41.053 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-04-26T04:30:41.151 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-04-26T04:30:41.187 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-04-26T04:30:41.310 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:30:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:30:41.464 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 28/33 2024-04-26T04:30:41.489 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:30:41.515 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 29/33 2024-04-26T04:30:41.515 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:41.516 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-26T04:30:41.516 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:30:41.516 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-26T04:30:41.516 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:41.571 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:41.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:30:42.295 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:08 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:42.422 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout:====================================================================================== 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout:====================================================================================== 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:30:42.424 INFO:teuthology.orchestra.run.smithi055.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout:====================================================================================== 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout:Install 12 Packages 2024-04-26T04:30:42.425 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:42.426 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 38 M 2024-04-26T04:30:42.426 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 209 M 2024-04-26T04:30:42.426 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:42.915 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:30:42.942 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 30/33 2024-04-26T04:30:42.943 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:42.943 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-26T04:30:42.943 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:30:42.943 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-26T04:30:42.943 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:42.987 INFO:teuthology.orchestra.run.smithi055.stdout:(1/12): libgfortran-11.4.1-3.el9.x86_64.rpm 5.0 MB/s | 803 kB 00:00 2024-04-26T04:30:43.046 INFO:teuthology.orchestra.run.smithi055.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 980 kB/s | 33 kB 00:00 2024-04-26T04:30:43.138 INFO:teuthology.orchestra.run.smithi055.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 33 MB/s | 3.0 MB 00:00 2024-04-26T04:30:43.171 INFO:teuthology.orchestra.run.smithi055.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 545 kB/s | 18 kB 00:00 2024-04-26T04:30:43.205 INFO:teuthology.orchestra.run.smithi055.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 1.1 MB/s | 35 kB 00:00 2024-04-26T04:30:43.333 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:30:43.347 INFO:teuthology.orchestra.run.smithi055.stdout:(6/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 33 MB/s | 4.7 MB 00:00 2024-04-26T04:30:43.356 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 31/33 2024-04-26T04:30:43.356 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:43.356 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-26T04:30:43.357 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:30:43.357 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-26T04:30:43.357 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:43.389 INFO:teuthology.orchestra.run.smithi055.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 5.8 MB/s | 244 kB 00:00 2024-04-26T04:30:43.489 INFO:teuthology.orchestra.run.smithi055.stdout:(8/12): ceph-mgr-diskprediction-local-17.2.7-90 11 MB/s | 7.4 MB 00:00 2024-04-26T04:30:43.567 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:30:43.590 INFO:teuthology.orchestra.run.smithi055.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 25 MB/s | 5.1 MB 00:00 2024-04-26T04:30:43.591 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 32/33 2024-04-26T04:30:43.591 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:43.591 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-26T04:30:43.591 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:30:43.591 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-26T04:30:43.591 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:43.655 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:30:43.731 INFO:teuthology.orchestra.run.smithi055.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.0 MB/s | 250 kB 00:00 2024-04-26T04:30:43.906 INFO:teuthology.orchestra.run.smithi055.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 182 kB/s | 195 kB 00:01 2024-04-26T04:30:44.057 INFO:teuthology.orchestra.run.smithi055.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-04-26T04:30:44.060 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:44.060 INFO:teuthology.orchestra.run.smithi055.stdout:Total 23 MB/s | 38 MB 00:01 2024-04-26T04:30:44.419 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:44.447 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:44.448 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:44.885 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:44.885 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:45.385 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 33/33 2024-04-26T04:30:45.385 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/33 2024-04-26T04:30:45.385 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/33 2024-04-26T04:30:45.385 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 3/33 2024-04-26T04:30:45.386 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 4/33 2024-04-26T04:30:45.386 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 5/33 2024-04-26T04:30:45.386 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.n 6/33 2024-04-26T04:30:45.386 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-04-26T04:30:45.386 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-04-26T04:30:45.386 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-04-26T04:30:45.387 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-04-26T04:30:45.387 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-04-26T04:30:45.387 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 12/33 2024-04-26T04:30:45.387 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-04-26T04:30:45.387 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-04-26T04:30:45.387 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-26T04:30:45.388 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-26T04:30:45.389 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-26T04:30:45.389 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-26T04:30:45.389 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-26T04:30:45.389 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-26T04:30:45.389 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-26T04:30:45.389 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-26T04:30:45.390 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-26T04:30:45.390 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-26T04:30:45.390 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-26T04:30:45.390 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-26T04:30:45.669 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:45.745 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:30:45.789 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:30:45.893 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:30:45.932 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:30:46.117 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:30:46.155 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:30:46.235 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mds-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-modules-core-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:46.260 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mon-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-osd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-26T04:30:46.261 INFO:teuthology.orchestra.run.smithi038.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-ply-3.11-14.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-26T04:30:46.262 INFO:teuthology.orchestra.run.smithi038.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-26T04:30:46.263 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:46.263 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:46.613 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-base 2024-04-26T04:30:46.915 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:30:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:30:47.144 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:52 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:47.225 INFO:teuthology.orchestra.run.smithi038.stdout:Package ceph-base-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:30:47.268 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:47.269 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:30:47.269 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:47.341 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install cephadm 2024-04-26T04:30:47.867 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:52 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:47.991 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout: cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 82 k 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:30:47.992 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:47.993 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:30:47.993 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:47.993 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 82 k 2024-04-26T04:30:47.993 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 372 k 2024-04-26T04:30:47.993 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:30:48.207 INFO:teuthology.orchestra.run.smithi038.stdout:cephadm-17.2.7-904.gd8c6b0a3.el9.noarch.rpm 383 kB/s | 82 kB 00:00 2024-04-26T04:30:48.208 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:48.208 INFO:teuthology.orchestra.run.smithi038.stdout:Total 380 kB/s | 82 kB 00:00 2024-04-26T04:30:48.208 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:30:48.214 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:30:48.214 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:30:48.216 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:30:48.216 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:30:48.298 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:30:48.339 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:30:48.766 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:48.863 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:49.174 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:49.391 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/1 2024-04-26T04:30:49.391 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:49.391 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:30:49.391 INFO:teuthology.orchestra.run.smithi038.stdout: cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:49.392 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:49.392 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:49.416 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:30:49.446 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:30:49.570 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-immutable-object-cache 2024-04-26T04:30:50.102 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:55 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:50.226 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:50.227 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repo Size 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 137 k 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 137 k 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 379 k 2024-04-26T04:30:50.228 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:30:50.531 INFO:teuthology.orchestra.run.smithi038.stdout:ceph-immutable-object-cache-17.2.7-904.gd8c6b0a 453 kB/s | 137 kB 00:00 2024-04-26T04:30:50.532 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:50.532 INFO:teuthology.orchestra.run.smithi038.stdout:Total 450 kB/s | 137 kB 00:00 2024-04-26T04:30:50.532 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:30:50.540 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:30:50.540 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:30:50.600 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:30:50.600 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:30:50.756 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:30:50.841 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:30:50.867 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:30:50.867 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:30:50.867 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-26T04:30:50.867 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:51.734 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.e 1/1 2024-04-26T04:30:51.734 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:51.734 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:30:51.734 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-immutable-object-cache-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:30:51.734 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:51.734 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:51.917 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-mgr 2024-04-26T04:30:52.001 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:30:52.002 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:30:52.397 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:30:52.398 INFO:teuthology.orchestra.run.smithi055.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:52.399 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:52.447 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:57 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:52.529 INFO:teuthology.orchestra.run.smithi038.stdout:Package ceph-mgr-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:30:52.572 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:52.573 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:30:52.573 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:52.655 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-mgr-dashboard 2024-04-26T04:30:52.726 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:30:53.214 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:00:58 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:53.253 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:19 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:53.339 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-dashboard noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 3.4 M 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-grafana-dashboards noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 24 k 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-prometheus-alerts noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 15 k 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-26T04:30:53.340 INFO:teuthology.orchestra.run.smithi038.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout:Installing weak dependencies: 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout:Install 7 Packages 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:53.341 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 3.7 M 2024-04-26T04:30:53.342 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 80 M 2024-04-26T04:30:53.342 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:30:53.380 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:53.381 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:30:53.382 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout:Install 13 Packages 2024-04-26T04:30:53.383 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:53.384 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 2.2 M 2024-04-26T04:30:53.384 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 26 M 2024-04-26T04:30:53.384 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:30:53.656 INFO:teuthology.orchestra.run.smithi055.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 158 kB/s | 19 kB 00:00 2024-04-26T04:30:53.882 INFO:teuthology.orchestra.run.smithi055.stdout:(2/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 990 kB/s | 222 kB 00:00 2024-04-26T04:30:53.919 INFO:teuthology.orchestra.run.smithi038.stdout:(1/7): ceph-prometheus-alerts-17.2.7-904.gd8c6b 76 kB/s | 15 kB 00:00 2024-04-26T04:30:53.970 INFO:teuthology.orchestra.run.smithi038.stdout:(2/7): ceph-grafana-dashboards-17.2.7-904.gd8c6 100 kB/s | 24 kB 00:00 2024-04-26T04:30:54.015 INFO:teuthology.orchestra.run.smithi055.stdout:(3/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 99 kB/s | 47 kB 00:00 2024-04-26T04:30:54.099 INFO:teuthology.orchestra.run.smithi055.stdout:(4/13): python3-jsonpatch-1.21-16.el9.noarch.rp 45 kB/s | 26 kB 00:00 2024-04-26T04:30:54.153 INFO:teuthology.orchestra.run.smithi038.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 39 kB/s | 9.0 kB 00:00 2024-04-26T04:30:54.270 INFO:teuthology.orchestra.run.smithi038.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 149 kB/s | 41 kB 00:00 2024-04-26T04:30:54.295 INFO:teuthology.orchestra.run.smithi038.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 216 kB/s | 31 kB 00:00 2024-04-26T04:30:54.454 INFO:teuthology.orchestra.run.smithi038.stdout:(6/7): ceph-mgr-dashboard-17.2.7-904.gd8c6b0a3. 4.6 MB/s | 3.4 MB 00:00 2024-04-26T04:30:54.479 INFO:teuthology.orchestra.run.smithi038.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 904 kB/s | 188 kB 00:00 2024-04-26T04:30:54.483 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:54.483 INFO:teuthology.orchestra.run.smithi038.stdout:Total 3.2 MB/s | 3.7 MB 00:01 2024-04-26T04:30:54.483 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:30:54.492 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:30:54.492 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:30:54.558 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:30:54.559 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:30:54.574 INFO:teuthology.orchestra.run.smithi055.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 230 kB/s | 159 kB 00:00 2024-04-26T04:30:54.783 INFO:teuthology.orchestra.run.smithi055.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 363 kB/s | 279 kB 00:00 2024-04-26T04:30:54.808 INFO:teuthology.orchestra.run.smithi055.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 76 kB/s | 54 kB 00:00 2024-04-26T04:30:54.834 INFO:teuthology.orchestra.run.smithi055.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 124 kB/s | 32 kB 00:00 2024-04-26T04:30:54.861 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:30:54.922 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-26T04:30:54.957 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-26T04:30:55.005 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-26T04:30:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 62 kB/s | 15 kB 00:00 2024-04-26T04:30:55.043 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-26T04:30:55.072 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 5/7 2024-04-26T04:30:55.170 INFO:teuthology.orchestra.run.smithi055.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 417 kB/s | 59 kB 00:00 2024-04-26T04:30:55.201 INFO:teuthology.orchestra.run.smithi055.stdout:(11/13): python3-kubernetes-26.1.0-2.el9.noarch 3.3 MB/s | 1.0 MB 00:00 2024-04-26T04:30:55.226 INFO:teuthology.orchestra.run.smithi055.stdout:(12/13): python3-google-auth-2.29.0-1.el9.noarc 493 kB/s | 206 kB 00:00 2024-04-26T04:30:55.285 INFO:teuthology.orchestra.run.smithi055.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 768 kB/s | 90 kB 00:00 2024-04-26T04:30:55.290 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:30:55.290 INFO:teuthology.orchestra.run.smithi055.stdout:Total 1.2 MB/s | 2.2 MB 00:01 2024-04-26T04:30:55.360 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:30:55.377 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:30:55.377 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:30:55.471 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:30:55.473 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:30:56.136 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 6/7 2024-04-26T04:30:56.184 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:30:56.195 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:30:56.375 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:30:56.488 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:30:56.544 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:30:56.580 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:30:56.621 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:30:56.718 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:30:56.829 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:30:56.881 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:30:57.289 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 7/7 2024-04-26T04:30:57.289 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.n 1/7 2024-04-26T04:30:57.289 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2/7 2024-04-26T04:30:57.289 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.no 3/7 2024-04-26T04:30:57.289 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-26T04:30:57.289 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-26T04:30:57.290 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-26T04:30:57.584 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-26T04:30:57.584 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:57.584 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:30:57.584 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-grafana-dashboards-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:57.584 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-dashboard-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-prometheus-alerts-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:57.585 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:30:57.641 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:30:57.686 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:30:57.720 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:30:57.766 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:30:57.783 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-26T04:30:57.789 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:30:58.321 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:03 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:30:58.447 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:30:58.449 INFO:teuthology.orchestra.run.smithi038.stdout:====================================================================================== 2024-04-26T04:30:58.449 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:30:58.449 INFO:teuthology.orchestra.run.smithi038.stdout:====================================================================================== 2024-04-26T04:30:58.449 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:30:58.449 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 7.4 M 2024-04-26T04:30:58.450 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:30:58.450 INFO:teuthology.orchestra.run.smithi038.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-04-26T04:30:58.450 INFO:teuthology.orchestra.run.smithi038.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-04-26T04:30:58.450 INFO:teuthology.orchestra.run.smithi038.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-04-26T04:30:58.450 INFO:teuthology.orchestra.run.smithi038.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-26T04:30:58.450 INFO:teuthology.orchestra.run.smithi038.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-04-26T04:30:58.451 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:58.452 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:30:58.452 INFO:teuthology.orchestra.run.smithi038.stdout:====================================================================================== 2024-04-26T04:30:58.452 INFO:teuthology.orchestra.run.smithi038.stdout:Install 12 Packages 2024-04-26T04:30:58.452 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:30:58.453 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 38 M 2024-04-26T04:30:58.453 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 209 M 2024-04-26T04:30:58.454 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:30:58.652 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:30:58.653 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:30:58.653 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:30:58.653 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:30:58.653 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:30:58.653 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:30:58.654 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:30:58.957 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:30:58.957 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:58.957 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:30:58.957 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:30:58.957 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:30:58.957 INFO:teuthology.orchestra.run.smithi055.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:58.958 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:30:59.105 INFO:teuthology.orchestra.run.smithi038.stdout:(1/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.3 MB/s | 803 kB 00:00 2024-04-26T04:30:59.138 INFO:teuthology.orchestra.run.smithi038.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 980 kB/s | 33 kB 00:00 2024-04-26T04:30:59.176 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:30:59.322 INFO:teuthology.orchestra.run.smithi038.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 16 MB/s | 3.0 MB 00:00 2024-04-26T04:30:59.380 INFO:teuthology.orchestra.run.smithi038.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 313 kB/s | 18 kB 00:00 2024-04-26T04:30:59.431 INFO:teuthology.orchestra.run.smithi038.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 703 kB/s | 35 kB 00:00 2024-04-26T04:30:59.706 INFO:teuthology.orchestra.run.smithi038.stdout:(6/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 17 MB/s | 4.7 MB 00:00 2024-04-26T04:30:59.707 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:25 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:30:59.765 INFO:teuthology.orchestra.run.smithi038.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 4.1 MB/s | 244 kB 00:00 2024-04-26T04:30:59.833 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:30:59.833 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:59.833 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout:Install 6 Packages 2024-04-26T04:30:59.834 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:30:59.835 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 7.0 M 2024-04-26T04:30:59.835 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 33 M 2024-04-26T04:30:59.835 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:00.023 INFO:teuthology.orchestra.run.smithi038.stdout:(8/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 20 MB/s | 5.1 MB 00:00 2024-04-26T04:31:00.049 INFO:teuthology.orchestra.run.smithi038.stdout:(9/12): libquadmath-11.4.1-3.el9.x86_64.rpm 165 kB/s | 195 kB 00:01 2024-04-26T04:31:00.082 INFO:teuthology.orchestra.run.smithi038.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 4.2 MB/s | 250 kB 00:00 2024-04-26T04:31:00.741 INFO:teuthology.orchestra.run.smithi038.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 23 MB/s | 16 MB 00:00 2024-04-26T04:31:00.862 INFO:teuthology.orchestra.run.smithi055.stdout:(1/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 488 kB/s | 123 kB 00:00 2024-04-26T04:31:01.046 INFO:teuthology.orchestra.run.smithi055.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-04-26T04:31:01.191 INFO:teuthology.orchestra.run.smithi038.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-9 3.2 MB/s | 7.4 MB 00:02 2024-04-26T04:31:01.194 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:01.194 INFO:teuthology.orchestra.run.smithi038.stdout:Total 14 MB/s | 38 MB 00:02 2024-04-26T04:31:01.271 INFO:teuthology.orchestra.run.smithi055.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.3 MB/s | 548 kB 00:00 2024-04-26T04:31:01.296 INFO:teuthology.orchestra.run.smithi055.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 232 kB/s | 58 kB 00:00 2024-04-26T04:31:01.330 INFO:teuthology.orchestra.run.smithi055.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 871 kB/s | 51 kB 00:00 2024-04-26T04:31:01.557 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:01.585 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:01.585 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:01.680 INFO:teuthology.orchestra.run.smithi055.stdout:(6/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 231 kB/s | 248 kB 00:01 2024-04-26T04:31:01.684 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:01.684 INFO:teuthology.orchestra.run.smithi055.stdout:Total 3.8 MB/s | 7.0 MB 00:01 2024-04-26T04:31:01.760 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:01.774 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:01.774 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:01.876 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:01.876 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:02.017 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:02.017 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:02.198 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:02.362 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:31:02.433 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:31:02.801 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:02.863 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-04-26T04:31:02.907 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:31:03.005 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:31:03.013 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:31:03.059 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-04-26T04:31:03.097 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:31:03.159 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:31:03.185 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:31:03.245 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-26T04:31:03.274 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-26T04:31:03.354 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-26T04:31:04.048 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-26T04:31:04.123 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:31:04.123 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:31:04.123 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:31:04.123 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:31:04.124 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:31:04.124 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:31:04.175 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-26T04:31:04.365 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:31:04.365 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:04.366 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:04.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-fuse 2024-04-26T04:31:05.157 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:31 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:05.282 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:31:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout:Install 2 Packages 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 892 k 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 2.6 M 2024-04-26T04:31:05.284 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:05.487 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:31:05.507 INFO:teuthology.orchestra.run.smithi055.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 643 kB/s | 80 kB 00:00 2024-04-26T04:31:05.698 INFO:teuthology.orchestra.run.smithi055.stdout:(2/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 2.5 MB/s | 812 kB 00:00 2024-04-26T04:31:05.700 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:05.700 INFO:teuthology.orchestra.run.smithi055.stdout:Total 2.1 MB/s | 892 kB 00:00 2024-04-26T04:31:05.712 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:05.734 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:05.735 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:05.816 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:05.816 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:06.000 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:06.131 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:31:06.249 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:31:06.585 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-26T04:31:06.625 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:31:06.925 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:31:06.926 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:31:07.171 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:31:07.172 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:07.172 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:07.172 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:31:07.172 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:07.172 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:07.424 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install librados-devel 2024-04-26T04:31:07.955 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:33 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:08.080 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:08.080 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 126 k 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 448 k 2024-04-26T04:31:08.081 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:08.399 INFO:teuthology.orchestra.run.smithi055.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 398 kB/s | 126 kB 00:00 2024-04-26T04:31:08.400 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:08.400 INFO:teuthology.orchestra.run.smithi055.stdout:Total 396 kB/s | 126 kB 00:00 2024-04-26T04:31:08.400 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:08.406 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:08.406 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:08.472 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:08.472 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:08.627 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:08.723 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:09.094 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 12/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0 1/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-04-26T04:31:09.105 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-04-26T04:31:09.316 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:09.317 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:09.317 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:09.317 INFO:teuthology.orchestra.run.smithi055.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:09.317 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:09.317 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:09.517 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-diskprediction-local-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-04-26T04:31:09.518 INFO:teuthology.orchestra.run.smithi038.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-26T04:31:09.519 INFO:teuthology.orchestra.run.smithi038.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-26T04:31:09.519 INFO:teuthology.orchestra.run.smithi038.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-26T04:31:09.519 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:09.519 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:09.557 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install libcephfs2 2024-04-26T04:31:09.875 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-mgr-rook 2024-04-26T04:31:10.087 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:10.168 INFO:teuthology.orchestra.run.smithi055.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:10.213 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:10.214 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:10.214 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:10.297 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install libcephfs-devel 2024-04-26T04:31:10.412 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:15 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:10.537 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-rook noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 47 k 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: python3-google-auth noarch 1:2.29.0-1.el9 epel 206 k 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-04-26T04:31:10.539 INFO:teuthology.orchestra.run.smithi038.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout:Install 13 Packages 2024-04-26T04:31:10.540 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:10.541 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 2.2 M 2024-04-26T04:31:10.541 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 26 M 2024-04-26T04:31:10.541 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:10.847 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:10.860 INFO:teuthology.orchestra.run.smithi038.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 177 kB/s | 19 kB 00:00 2024-04-26T04:31:10.952 INFO:teuthology.orchestra.run.smithi038.stdout:(2/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.4 MB/s | 222 kB 00:00 2024-04-26T04:31:10.971 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:10.971 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 25 k 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 110 k 2024-04-26T04:31:10.972 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:10.994 INFO:teuthology.orchestra.run.smithi038.stdout:(3/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.7 MB/s | 159 kB 00:00 2024-04-26T04:31:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:(4/13): ceph-mgr-rook-17.2.7-904.gd8c6b0a3.el9. 172 kB/s | 47 kB 00:00 2024-04-26T04:31:11.061 INFO:teuthology.orchestra.run.smithi038.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 279 kB 00:00 2024-04-26T04:31:11.087 INFO:teuthology.orchestra.run.smithi038.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 920 kB/s | 54 kB 00:00 2024-04-26T04:31:11.246 INFO:teuthology.orchestra.run.smithi055.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 91 kB/s | 25 kB 00:00 2024-04-26T04:31:11.247 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:11.247 INFO:teuthology.orchestra.run.smithi055.stdout:Total 91 kB/s | 25 kB 00:00 2024-04-26T04:31:11.247 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:11.251 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:11.251 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:11.252 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:11.253 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:11.303 INFO:teuthology.orchestra.run.smithi038.stdout:(7/13): python3-jsonpatch-1.21-16.el9.noarch.rp 46 kB/s | 26 kB 00:00 2024-04-26T04:31:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:11.412 INFO:teuthology.orchestra.run.smithi038.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 46 kB/s | 15 kB 00:00 2024-04-26T04:31:11.437 INFO:teuthology.orchestra.run.smithi038.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 86 kB/s | 32 kB 00:00 2024-04-26T04:31:11.453 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:11.646 INFO:teuthology.orchestra.run.smithi038.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 284 kB/s | 59 kB 00:00 2024-04-26T04:31:11.800 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:11.974 INFO:teuthology.orchestra.run.smithi038.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 276 kB/s | 90 kB 00:00 2024-04-26T04:31:11.996 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:11.996 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:11.996 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:11.996 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:11.996 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:11.997 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:12.179 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install librados2 2024-04-26T04:31:12.205 INFO:teuthology.orchestra.run.smithi038.stdout:(12/13): python3-google-auth-2.29.0-1.el9.noarc 228 kB/s | 206 kB 00:00 2024-04-26T04:31:12.712 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:12.794 INFO:teuthology.orchestra.run.smithi055.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:12.836 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:12.837 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:12.837 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:12.913 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install librbd1 2024-04-26T04:31:13.445 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:13.526 INFO:teuthology.orchestra.run.smithi055.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:13.568 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:13.569 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:13.570 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:13.636 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-rados 2024-04-26T04:31:13.772 INFO:teuthology.orchestra.run.smithi038.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 443 kB/s | 1.0 MB 00:02 2024-04-26T04:31:13.774 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:13.774 INFO:teuthology.orchestra.run.smithi038.stdout:Total 696 kB/s | 2.2 MB 00:03 2024-04-26T04:31:13.844 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:13.861 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:13.861 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:13.960 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:13.961 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:14.167 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:14.249 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:14.293 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:14.294 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:14.294 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:14.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-rgw 2024-04-26T04:31:14.547 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:14.718 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-26T04:31:14.784 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-26T04:31:14.841 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-26T04:31:14.875 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-26T04:31:14.900 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:40 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:14.917 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-26T04:31:14.981 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:15.007 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-26T04:31:15.024 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:15.025 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:15.025 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:15.093 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-cephfs 2024-04-26T04:31:15.119 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-google-auth-1:2.29.0-1.el9.noarch 7/13 2024-04-26T04:31:15.170 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-26T04:31:15.631 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:15.713 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:15.756 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:15.758 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:15.758 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:15.830 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-rbd 2024-04-26T04:31:15.934 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-26T04:31:15.983 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-26T04:31:16.017 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-26T04:31:16.065 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-26T04:31:16.086 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:31:16.369 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:42 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:16.451 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:16.494 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:16.495 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2024-04-26T04:31:16.495 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:16.571 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install rbd-fuse 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 13/13 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/13 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-26T04:31:16.840 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-26T04:31:16.841 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-26T04:31:16.841 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-26T04:31:16.841 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-26T04:31:16.841 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-google-auth-1:2.29.0-1.el9.noarch 10/13 2024-04-26T04:31:16.841 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-26T04:31:16.841 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-26T04:31:17.107 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:43 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:17.182 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-26T04:31:17.182 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:17.182 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-rook-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-google-auth-1:2.29.0-1.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-26T04:31:17.183 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:17.184 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:17.231 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 82 k 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 222 k 2024-04-26T04:31:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:17.394 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-mgr-cephadm 2024-04-26T04:31:17.458 INFO:teuthology.orchestra.run.smithi055.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 363 kB/s | 82 kB 00:00 2024-04-26T04:31:17.459 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:17.459 INFO:teuthology.orchestra.run.smithi055.stdout:Total 361 kB/s | 82 kB 00:00 2024-04-26T04:31:17.459 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:17.466 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:17.466 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:17.532 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:17.532 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:17.677 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:17.773 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:17.934 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:22 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:18.058 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:18.058 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-cephadm noarch 2:17.2.7-904.gd8c6b0a3.el9 ceph-noarch 123 k 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-04-26T04:31:18.059 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout:Install 6 Packages 2024-04-26T04:31:18.060 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:18.061 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 7.0 M 2024-04-26T04:31:18.061 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 33 M 2024-04-26T04:31:18.061 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:18.138 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:18.357 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:18.358 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:18.358 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:18.358 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:18.358 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:18.358 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:18.536 INFO:teuthology.orchestra.run.smithi038.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.4 MB/s | 248 kB 00:00 2024-04-26T04:31:18.602 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install rbd-mirror 2024-04-26T04:31:18.721 INFO:teuthology.orchestra.run.smithi038.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 17 MB/s | 6.0 MB 00:00 2024-04-26T04:31:18.811 INFO:teuthology.orchestra.run.smithi038.stdout:(3/6): ceph-mgr-cephadm-17.2.7-904.gd8c6b0a3.el 277 kB/s | 123 kB 00:00 2024-04-26T04:31:18.920 INFO:teuthology.orchestra.run.smithi038.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 288 kB/s | 58 kB 00:00 2024-04-26T04:31:18.945 INFO:teuthology.orchestra.run.smithi038.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.3 MB/s | 548 kB 00:00 2024-04-26T04:31:18.996 INFO:teuthology.orchestra.run.smithi038.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 277 kB/s | 51 kB 00:00 2024-04-26T04:31:18.998 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:18.998 INFO:teuthology.orchestra.run.smithi038.stdout:Total 7.4 MB/s | 7.0 MB 00:00 2024-04-26T04:31:19.072 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:19.084 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:19.084 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:19.139 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:45 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:19.186 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:19.186 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:19.263 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:19.264 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:31:19.265 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:19.265 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 2.9 M 2024-04-26T04:31:19.265 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 12 M 2024-04-26T04:31:19.265 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:19.509 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:19.678 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-26T04:31:19.749 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-26T04:31:20.139 INFO:teuthology.orchestra.run.smithi055.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 3.4 MB/s | 2.9 MB 00:00 2024-04-26T04:31:20.140 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:20.140 INFO:teuthology.orchestra.run.smithi055.stdout:Total 3.4 MB/s | 2.9 MB 00:00 2024-04-26T04:31:20.140 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:20.150 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:20.150 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:20.233 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:20.233 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:20.308 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-26T04:31:20.395 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-26T04:31:20.458 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-26T04:31:20.485 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:31:20.671 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:20.823 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:20.850 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:20.850 INFO:teuthology.orchestra.run.smithi055.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:20.851 INFO:teuthology.orchestra.run.smithi055.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:31:20.851 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:31:20.851 INFO:teuthology.orchestra.run.smithi055.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:31:20.851 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:21.237 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 6/6 2024-04-26T04:31:21.238 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 1/6 2024-04-26T04:31:21.238 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-26T04:31:21.238 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-26T04:31:21.238 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-26T04:31:21.238 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-26T04:31:21.496 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-mgr-cephadm-2:17.2.7-904.gd8c6b0a3.el9.noarch 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:21.497 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:21.727 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install ceph-fuse 2024-04-26T04:31:21.792 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:21.792 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:21.792 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:21.792 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:21.792 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:21.792 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:22.025 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install rbd-nbd 2024-04-26T04:31:22.264 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:27 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:22.389 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:22.389 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:22.389 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repository Size 2024-04-26T04:31:22.389 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 812 k 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout:Installing dependencies: 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout: fuse x86_64 2.9.9-15.el9 baseos 80 k 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout:Install 2 Packages 2024-04-26T04:31:22.390 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:22.391 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 892 k 2024-04-26T04:31:22.391 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 2.6 M 2024-04-26T04:31:22.391 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:22.564 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:48 ago on Fri 26 Apr 2024 04:29:34 AM UTC. 2024-04-26T04:31:22.665 INFO:teuthology.orchestra.run.smithi038.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 599 kB/s | 80 kB 00:00 2024-04-26T04:31:22.688 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2024-04-26T04:31:22.689 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:22.689 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:22.689 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:22.689 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2024-04-26T04:31:22.689 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 165 k 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 477 k 2024-04-26T04:31:22.690 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2024-04-26T04:31:23.070 INFO:teuthology.orchestra.run.smithi055.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 434 kB/s | 165 kB 00:00 2024-04-26T04:31:23.071 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:23.071 INFO:teuthology.orchestra.run.smithi055.stdout:Total 432 kB/s | 165 kB 00:00 2024-04-26T04:31:23.071 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2024-04-26T04:31:23.080 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2024-04-26T04:31:23.080 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2024-04-26T04:31:23.106 INFO:teuthology.orchestra.run.smithi038.stdout:(2/2): ceph-fuse-17.2.7-904.gd8c6b0a3.el9.x86_6 1.4 MB/s | 812 kB 00:00 2024-04-26T04:31:23.108 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:23.108 INFO:teuthology.orchestra.run.smithi038.stdout:Total 1.2 MB/s | 892 kB 00:00 2024-04-26T04:31:23.120 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:23.144 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:23.144 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:23.147 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2024-04-26T04:31:23.147 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2024-04-26T04:31:23.226 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:23.226 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:23.308 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2024-04-26T04:31:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:23.417 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:23.556 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-26T04:31:23.625 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:31:23.804 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:24.034 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:24.035 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:24.035 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2024-04-26T04:31:24.035 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:24.035 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:31:24.035 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2024-04-26T04:31:24.241 DEBUG:teuthology.parallel:result is None 2024-04-26T04:31:24.363 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2/2 2024-04-26T04:31:24.363 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/2 2024-04-26T04:31:24.632 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-26T04:31:24.632 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:24.632 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:24.633 INFO:teuthology.orchestra.run.smithi038.stdout: ceph-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-26T04:31:24.633 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:24.633 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:24.887 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install librados-devel 2024-04-26T04:31:25.417 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:30 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:25.541 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repo Size 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout: librados-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 126 k 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:25.542 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 126 k 2024-04-26T04:31:25.543 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 448 k 2024-04-26T04:31:25.543 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:25.861 INFO:teuthology.orchestra.run.smithi038.stdout:librados-devel-17.2.7-904.gd8c6b0a3.el9.x86_64. 397 kB/s | 126 kB 00:00 2024-04-26T04:31:25.862 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:25.862 INFO:teuthology.orchestra.run.smithi038.stdout:Total 395 kB/s | 126 kB 00:00 2024-04-26T04:31:25.862 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:25.868 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:25.868 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:25.934 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:25.935 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:26.093 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:26.193 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:26.557 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:26.829 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:26.829 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:26.829 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:26.829 INFO:teuthology.orchestra.run.smithi038.stdout: librados-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:26.829 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:26.830 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:27.036 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install libcephfs2 2024-04-26T04:31:27.570 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:32 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:27.652 INFO:teuthology.orchestra.run.smithi038.stdout:Package libcephfs2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:27.695 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:27.696 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:27.697 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:27.766 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install libcephfs-devel 2024-04-26T04:31:28.307 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:33 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:28.432 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:28.432 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repo Size 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout: libcephfs-devel x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 25 k 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 25 k 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 110 k 2024-04-26T04:31:28.433 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:28.690 INFO:teuthology.orchestra.run.smithi038.stdout:libcephfs-devel-17.2.7-904.gd8c6b0a3.el9.x86_64 97 kB/s | 25 kB 00:00 2024-04-26T04:31:28.691 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:28.691 INFO:teuthology.orchestra.run.smithi038.stdout:Total 97 kB/s | 25 kB 00:00 2024-04-26T04:31:28.691 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:28.694 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:28.695 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:28.696 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:28.696 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:28.775 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:28.880 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:29.241 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:29.453 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:29.453 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:29.453 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:29.453 INFO:teuthology.orchestra.run.smithi038.stdout: libcephfs-devel-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:29.453 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:29.453 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:29.622 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install librados2 2024-04-26T04:31:30.165 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:30.245 INFO:teuthology.orchestra.run.smithi038.stdout:Package librados2-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:30.288 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:30.289 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:30.289 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:30.348 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install librbd1 2024-04-26T04:31:30.885 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:35 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:30.967 INFO:teuthology.orchestra.run.smithi038.stdout:Package librbd1-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:31.009 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:31.010 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:31.010 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:31.077 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install python3-rados 2024-04-26T04:31:31.606 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:36 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:31.688 INFO:teuthology.orchestra.run.smithi038.stdout:Package python3-rados-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:31.731 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:31.732 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:31.732 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:31.793 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install python3-rgw 2024-04-26T04:31:32.334 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:37 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:32.416 INFO:teuthology.orchestra.run.smithi038.stdout:Package python3-rgw-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:32.459 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:32.460 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:32.460 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:32.525 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install python3-cephfs 2024-04-26T04:31:33.061 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:33.143 INFO:teuthology.orchestra.run.smithi038.stdout:Package python3-cephfs-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:33.186 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:33.187 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:33.187 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:33.252 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install python3-rbd 2024-04-26T04:31:33.792 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:38 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:33.874 INFO:teuthology.orchestra.run.smithi038.stdout:Package python3-rbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 is already installed. 2024-04-26T04:31:33.917 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:33.918 INFO:teuthology.orchestra.run.smithi038.stdout:Nothing to do. 2024-04-26T04:31:33.918 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:33.983 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install rbd-fuse 2024-04-26T04:31:34.519 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:39 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:34.643 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout: rbd-fuse x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 82 k 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:31:34.644 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:34.645 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 82 k 2024-04-26T04:31:34.645 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 222 k 2024-04-26T04:31:34.645 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:34.960 INFO:teuthology.orchestra.run.smithi038.stdout:rbd-fuse-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 260 kB/s | 82 kB 00:00 2024-04-26T04:31:34.961 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:34.961 INFO:teuthology.orchestra.run.smithi038.stdout:Total 258 kB/s | 82 kB 00:00 2024-04-26T04:31:34.961 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:34.969 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:34.969 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:35.034 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:35.034 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:35.183 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:35.291 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:35.620 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:35.861 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:35.861 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:35.861 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:35.861 INFO:teuthology.orchestra.run.smithi038.stdout: rbd-fuse-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:35.861 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:35.862 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:36.061 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install rbd-mirror 2024-04-26T04:31:36.597 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:41 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:36.725 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:36.725 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:36.725 INFO:teuthology.orchestra.run.smithi038.stdout: Package Arch Version Repo Size 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout: rbd-mirror x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 2.9 M 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 2.9 M 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 12 M 2024-04-26T04:31:36.726 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:37.748 INFO:teuthology.orchestra.run.smithi038.stdout:rbd-mirror-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 2.9 MB/s | 2.9 MB 00:01 2024-04-26T04:31:37.748 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:37.749 INFO:teuthology.orchestra.run.smithi038.stdout:Total 2.9 MB/s | 2.9 MB 00:01 2024-04-26T04:31:37.749 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:37.759 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:37.759 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:37.842 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:37.842 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:38.284 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:38.432 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:38.459 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:38.459 INFO:teuthology.orchestra.run.smithi038.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-26T04:31:38.459 INFO:teuthology.orchestra.run.smithi038.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-26T04:31:38.459 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:31:38.459 INFO:teuthology.orchestra.run.smithi038.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-26T04:31:38.460 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:39.403 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:39.403 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:39.403 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:39.403 INFO:teuthology.orchestra.run.smithi038.stdout: rbd-mirror-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:39.403 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:39.403 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:39.644 DEBUG:teuthology.orchestra.run.smithi038:> sudo yum -y install rbd-nbd 2024-04-26T04:31:40.186 INFO:teuthology.orchestra.run.smithi038.stdout:Last metadata expiration check: 0:01:45 ago on Fri 26 Apr 2024 04:29:55 AM UTC. 2024-04-26T04:31:40.310 INFO:teuthology.orchestra.run.smithi038.stdout:Dependencies resolved. 2024-04-26T04:31:40.311 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:40.311 INFO:teuthology.orchestra.run.smithi038.stdout: Package Architecture Version Repository Size 2024-04-26T04:31:40.311 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:Installing: 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout: rbd-nbd x86_64 2:17.2.7-904.gd8c6b0a3.el9 ceph 165 k 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction Summary 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:================================================================================ 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:Install 1 Package 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:Total download size: 165 k 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:Installed size: 477 k 2024-04-26T04:31:40.312 INFO:teuthology.orchestra.run.smithi038.stdout:Downloading Packages: 2024-04-26T04:31:40.702 INFO:teuthology.orchestra.run.smithi038.stdout:rbd-nbd-17.2.7-904.gd8c6b0a3.el9.x86_64.rpm 423 kB/s | 165 kB 00:00 2024-04-26T04:31:40.703 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:31:40.703 INFO:teuthology.orchestra.run.smithi038.stdout:Total 422 kB/s | 165 kB 00:00 2024-04-26T04:31:40.703 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction check 2024-04-26T04:31:40.712 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction check succeeded. 2024-04-26T04:31:40.712 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction test 2024-04-26T04:31:40.779 INFO:teuthology.orchestra.run.smithi038.stdout:Transaction test succeeded. 2024-04-26T04:31:40.779 INFO:teuthology.orchestra.run.smithi038.stdout:Running transaction 2024-04-26T04:31:40.933 INFO:teuthology.orchestra.run.smithi038.stdout: Preparing : 1/1 2024-04-26T04:31:41.059 INFO:teuthology.orchestra.run.smithi038.stdout: Installing : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:41.436 INFO:teuthology.orchestra.run.smithi038.stdout: Running scriptlet: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:41.686 INFO:teuthology.orchestra.run.smithi038.stdout: Verifying : rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 1/1 2024-04-26T04:31:41.686 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:41.686 INFO:teuthology.orchestra.run.smithi038.stdout:Installed: 2024-04-26T04:31:41.687 INFO:teuthology.orchestra.run.smithi038.stdout: rbd-nbd-2:17.2.7-904.gd8c6b0a3.el9.x86_64 2024-04-26T04:31:41.687 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:31:41.687 INFO:teuthology.orchestra.run.smithi038.stdout:Complete! 2024-04-26T04:31:41.915 DEBUG:teuthology.parallel:result is None 2024-04-26T04:31:41.915 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:31:41.915 INFO:teuthology.packaging:ref: None 2024-04-26T04:31:41.916 INFO:teuthology.packaging:tag: None 2024-04-26T04:31:41.916 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:31:41.916 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:31:41.916 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:31:42.168 DEBUG:teuthology.orchestra.run.smithi038:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:31:42.190 INFO:teuthology.orchestra.run.smithi038.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:31:42.190 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:31:42.190 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:31:42.192 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-26T04:31:42.192 INFO:teuthology.packaging:ref: None 2024-04-26T04:31:42.192 INFO:teuthology.packaging:tag: None 2024-04-26T04:31:42.192 INFO:teuthology.packaging:branch: quincy 2024-04-26T04:31:42.192 INFO:teuthology.packaging:sha1: b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:31:42.192 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-26T04:31:42.423 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-26T04:31:42.448 INFO:teuthology.orchestra.run.smithi055.stdout:17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:31:42.448 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-904.gd8c6b0a3.el9 2024-04-26T04:31:42.448 INFO:teuthology.task.install:The correct ceph version 17.2.7-904.gd8c6b0a3 is installed. 2024-04-26T04:31:42.450 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-26T04:31:42.450 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:31:42.450 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:31:42.479 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:31:42.479 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-26T04:31:42.517 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-26T04:31:42.517 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:31:42.517 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:31:42.549 DEBUG:teuthology.orchestra.run.smithi038:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:31:42.615 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:31:42.615 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/usr/bin/daemon-helper 2024-04-26T04:31:42.646 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-26T04:31:42.711 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-26T04:31:42.711 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:31:42.711 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:31:42.737 DEBUG:teuthology.orchestra.run.smithi038:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:31:42.802 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:31:42.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-26T04:31:42.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-26T04:31:42.898 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-26T04:31:42.898 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:31:42.898 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:31:42.923 DEBUG:teuthology.orchestra.run.smithi038:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:31:42.989 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:31:42.989 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/usr/bin/stdin-killer 2024-04-26T04:31:43.016 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-26T04:31:43.085 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-26T04:31:43.184 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b22e2ebdeb24376882b7bda2a7329c8cccc2276a'} 2024-04-26T04:31:43.184 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:31:43.184 INFO:tasks.cephadm:Cluster fsid is e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:31:43.185 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-26T04:31:43.185 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.38', 'mon.c': '[v2:172.21.15.38:3301,v1:172.21.15.38:6790]', 'mon.b': '172.21.15.55'} 2024-04-26T04:31:43.185 INFO:tasks.cephadm:First mon is mon.a on smithi038 2024-04-26T04:31:43.185 INFO:tasks.cephadm:First mgr is y 2024-04-26T04:31:43.185 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-26T04:31:43.185 DEBUG:teuthology.orchestra.run.smithi038:> sudo hostname $(hostname -s) 2024-04-26T04:31:43.211 DEBUG:teuthology.orchestra.run.smithi055:> sudo hostname $(hostname -s) 2024-04-26T04:31:43.239 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-26T04:31:43.240 DEBUG:teuthology.orchestra.run.smithi038:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:31:43.509 INFO:teuthology.orchestra.run.smithi038.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:31 /home/ubuntu/cephtest/cephadm 2024-04-26T04:31:43.509 DEBUG:teuthology.orchestra.run.smithi055:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-26T04:31:43.690 INFO:teuthology.orchestra.run.smithi055.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 26 04:31 /home/ubuntu/cephtest/cephadm 2024-04-26T04:31:43.690 DEBUG:teuthology.orchestra.run.smithi038:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:31:43.708 DEBUG:teuthology.orchestra.run.smithi055:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-26T04:31:43.738 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-26T04:31:43.738 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:31:43.751 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-26T04:31:45.266 INFO:teuthology.orchestra.run.smithi055.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:31:45.316 INFO:teuthology.orchestra.run.smithi038.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:32:48.045 INFO:teuthology.orchestra.run.smithi038.stdout:{ 2024-04-26T04:32:48.045 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:32:48.045 INFO:teuthology.orchestra.run.smithi038.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:32:48.045 INFO:teuthology.orchestra.run.smithi038.stdout: "repo_digests": [ 2024-04-26T04:32:48.046 INFO:teuthology.orchestra.run.smithi038.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:32:48.046 INFO:teuthology.orchestra.run.smithi038.stdout: ] 2024-04-26T04:32:48.046 INFO:teuthology.orchestra.run.smithi038.stdout:} 2024-04-26T04:32:51.031 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2024-04-26T04:32:51.031 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph_version": "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)", 2024-04-26T04:32:51.031 INFO:teuthology.orchestra.run.smithi055.stdout: "image_id": "0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43", 2024-04-26T04:32:51.031 INFO:teuthology.orchestra.run.smithi055.stdout: "repo_digests": [ 2024-04-26T04:32:51.032 INFO:teuthology.orchestra.run.smithi055.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17" 2024-04-26T04:32:51.032 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2024-04-26T04:32:51.032 INFO:teuthology.orchestra.run.smithi055.stdout:} 2024-04-26T04:32:51.053 DEBUG:teuthology.orchestra.run.smithi038:> sudo mkdir -p /etc/ceph 2024-04-26T04:32:51.086 DEBUG:teuthology.orchestra.run.smithi055:> sudo mkdir -p /etc/ceph 2024-04-26T04:32:51.116 DEBUG:teuthology.orchestra.run.smithi038:> sudo chmod 777 /etc/ceph 2024-04-26T04:32:51.156 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 777 /etc/ceph 2024-04-26T04:32:51.182 INFO:tasks.cephadm:Writing seed config... 2024-04-26T04:32:51.183 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-26T04:32:51.183 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-26T04:32:51.183 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-26T04:32:51.183 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-26T04:32:51.184 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-26T04:32:51.185 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:32:51.185 DEBUG:teuthology.orchestra.run.smithi038:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-26T04:32:51.213 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e22c1684-0385-11ef-bc93-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-26T04:32:51.214 DEBUG:teuthology.orchestra.run.smithi038:mon.a> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service 2024-04-26T04:32:51.260 DEBUG:teuthology.orchestra.run.smithi038:mgr.y> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service 2024-04-26T04:32:51.303 INFO:tasks.cephadm:Bootstrapping... 2024-04-26T04:32:51.303 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid e22c1684-0385-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.38 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:32:51.515 INFO:teuthology.orchestra.run.smithi038.stdout:-------------------------------------------------------------------------------- 2024-04-26T04:32:51.515 INFO:teuthology.orchestra.run.smithi038.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'e22c1684-0385-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.38', '--skip-admin-label'] 2024-04-26T04:32:51.540 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:32:51.541 INFO:teuthology.orchestra.run.smithi038.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-26T04:32:51.541 INFO:teuthology.orchestra.run.smithi038.stdout:Verifying podman|docker is present... 2024-04-26T04:32:51.573 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:32:51.573 INFO:teuthology.orchestra.run.smithi038.stdout:Verifying lvm2 is present... 2024-04-26T04:32:51.573 INFO:teuthology.orchestra.run.smithi038.stdout:Verifying time synchronization is in place... 2024-04-26T04:32:51.580 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:32:51.580 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:32:51.587 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:32:51.587 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout inactive 2024-04-26T04:32:51.594 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout enabled 2024-04-26T04:32:51.600 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout active 2024-04-26T04:32:51.600 INFO:teuthology.orchestra.run.smithi038.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:32:51.600 INFO:teuthology.orchestra.run.smithi038.stdout:Repeating the final host check... 2024-04-26T04:32:51.624 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-26T04:32:51.625 INFO:teuthology.orchestra.run.smithi038.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-26T04:32:51.625 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl is present 2024-04-26T04:32:51.625 INFO:teuthology.orchestra.run.smithi038.stdout:lvcreate is present 2024-04-26T04:32:51.631 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-26T04:32:51.632 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-26T04:32:51.638 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-26T04:32:51.638 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout inactive 2024-04-26T04:32:51.645 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout enabled 2024-04-26T04:32:51.651 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout active 2024-04-26T04:32:51.652 INFO:teuthology.orchestra.run.smithi038.stdout:Unit chronyd.service is enabled and running 2024-04-26T04:32:51.652 INFO:teuthology.orchestra.run.smithi038.stdout:Host looks OK 2024-04-26T04:32:51.652 INFO:teuthology.orchestra.run.smithi038.stdout:Cluster fsid: e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:32:51.652 INFO:teuthology.orchestra.run.smithi038.stdout:Acquiring lock 140602045756176 on /run/cephadm/e22c1684-0385-11ef-bc93-c7b262605968.lock 2024-04-26T04:32:51.652 INFO:teuthology.orchestra.run.smithi038.stdout:Lock 140602045756176 acquired on /run/cephadm/e22c1684-0385-11ef-bc93-c7b262605968.lock 2024-04-26T04:32:51.652 INFO:teuthology.orchestra.run.smithi038.stdout:Verifying IP 172.21.15.38 port 3300 ... 2024-04-26T04:32:51.653 INFO:teuthology.orchestra.run.smithi038.stdout:Verifying IP 172.21.15.38 port 6789 ... 2024-04-26T04:32:51.653 INFO:teuthology.orchestra.run.smithi038.stdout:Base mon IP(s) is [172.21.15.38:3300, 172.21.15.38:6789], mon addrv is [v2:172.21.15.38:3300,v1:172.21.15.38:6789] 2024-04-26T04:32:51.656 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.38 metric 100 2024-04-26T04:32:51.656 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.38 metric 100 2024-04-26T04:32:51.659 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-26T04:32:51.659 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-26T04:32:51.659 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-26T04:32:51.662 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-26T04:32:51.662 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-26T04:32:51.662 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:32:51.662 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-26T04:32:51.662 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:15fc/64 scope link noprefixroute 2024-04-26T04:32:51.662 INFO:teuthology.orchestra.run.smithi038.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-26T04:32:51.666 INFO:teuthology.orchestra.run.smithi038.stdout:Mon IP `172.21.15.38` is in CIDR network `172.21.0.0/20` 2024-04-26T04:32:51.666 INFO:teuthology.orchestra.run.smithi038.stdout:Mon IP `172.21.15.38` is in CIDR network `172.21.0.0/20` 2024-04-26T04:32:51.666 INFO:teuthology.orchestra.run.smithi038.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-26T04:32:51.666 INFO:teuthology.orchestra.run.smithi038.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-26T04:32:51.667 INFO:teuthology.orchestra.run.smithi038.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stdout 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stderr Getting image source signatures 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stderr Copying blob sha256:0e2af62137028b073c6ccdb8d4d7e0ccccc475ce5f6e225cb983e90813813045 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stderr Copying blob sha256:3d665f4c032644be9cae748af04bee69f2bfe96cdc3616dabe0ab17899633ab5 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stderr Copying config sha256:0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 2024-04-26T04:32:52.168 INFO:teuthology.orchestra.run.smithi038.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-26T04:32:53.050 INFO:teuthology.orchestra.run.smithi038.stdout:ceph: stdout ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:32:53.050 INFO:teuthology.orchestra.run.smithi038.stdout:Ceph version: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable) 2024-04-26T04:32:53.050 INFO:teuthology.orchestra.run.smithi038.stdout:Extracting ceph user uid/gid from container image... 2024-04-26T04:32:54.056 INFO:teuthology.orchestra.run.smithi038.stdout:stat: stdout 167 167 2024-04-26T04:32:54.056 INFO:teuthology.orchestra.run.smithi038.stdout:Creating initial keys... 2024-04-26T04:32:54.940 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph-authtool: stdout AQB2LitmCYiYFRAAWup4005al1uETSURP0vZ/g== 2024-04-26T04:32:55.864 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph-authtool: stdout AQB3Litmi86dDxAAC7aGWp4tRi5qebCIB/v7Ow== 2024-04-26T04:32:56.731 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph-authtool: stdout AQB4LitmyzzLChAAIzCG1cRzEKIw3K1ZpSShaQ== 2024-04-26T04:32:56.731 INFO:teuthology.orchestra.run.smithi038.stdout:Creating initial monmap... 2024-04-26T04:32:57.645 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:32:57.645 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-26T04:32:57.645 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout:monmaptool for a [v2:172.21.15.38:3300,v1:172.21.15.38:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout:setting min_mon_release = octopus 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/monmaptool: set fsid to e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:32:57.646 INFO:teuthology.orchestra.run.smithi038.stdout:Creating mon... 2024-04-26T04:32:58.493 INFO:teuthology.orchestra.run.smithi038.stdout:create mon.a on 2024-04-26T04:32:58.714 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-26T04:32:58.863 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-26T04:32:59.022 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e22c1684-0385-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-e22c1684-0385-11ef-bc93-c7b262605968.target. 2024-04-26T04:32:59.022 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-e22c1684-0385-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-e22c1684-0385-11ef-bc93-c7b262605968.target. 2024-04-26T04:32:59.215 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a 2024-04-26T04:32:59.215 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-26T04:32:59.392 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e22c1684-0385-11ef-bc93-c7b262605968.target.wants/ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-e22c1684-0385-11ef-bc93-c7b262605968@.service. 2024-04-26T04:32:59.760 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 podman[26062]: 2024-04-26 04:32:59.576144386 +0000 UTC m=+0.025516231 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:32:59.983 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:32:59.983 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout disabled 2024-04-26T04:32:59.991 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:32:59.991 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout inactive 2024-04-26T04:32:59.991 INFO:teuthology.orchestra.run.smithi038.stdout:firewalld.service is not enabled 2024-04-26T04:32:59.991 INFO:teuthology.orchestra.run.smithi038.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:32:59.992 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for mon to start... 2024-04-26T04:32:59.992 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for mon... 2024-04-26T04:33:00.104 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 podman[26062]: 2024-04-26 04:32:59.851457162 +0000 UTC m=+0.300828998 container init 30d1026b4fce06db7366917643aea8b02a5f33366c9d820c46f96eba8c709547 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux ) 2024-04-26T04:33:00.104 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 podman[26062]: 2024-04-26 04:32:59.854547062 +0000 UTC m=+0.303918884 container start 30d1026b4fce06db7366917643aea8b02a5f33366c9d820c46f96eba8c709547 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-26T04:33:00.104 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 ceph-mon[26083]: mkfs e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:00.104 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 ceph-mon[26083]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:33:00.104 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 bash[26062]: 30d1026b4fce06db7366917643aea8b02a5f33366c9d820c46f96eba8c709547 2024-04-26T04:33:00.104 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:32:59 smithi038 systemd[1]: Started Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:33:01.062 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout cluster: 2024-04-26T04:33:01.062 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout id: e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:01.062 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-26T04:33:01.062 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:01.062 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout services: 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.57634s) 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout data: 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout pgs: 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:mon is available 2024-04-26T04:33:01.063 INFO:teuthology.orchestra.run.smithi038.stdout:Assimilating anything we can from ceph.conf... 2024-04-26T04:33:01.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:00 smithi038 ceph-mon[26083]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:33:01.176 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:00 smithi038 ceph-mon[26083]: monmap e1: 1 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:33:01.176 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:00 smithi038 ceph-mon[26083]: fsmap 2024-04-26T04:33:01.176 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:00 smithi038 ceph-mon[26083]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:33:01.176 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:00 smithi038 ceph-mon[26083]: mgrmap e1: no daemons active 2024-04-26T04:33:01.176 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:00 smithi038 ceph-mon[26083]: from='client.? 172.21.15.38:0/1564694861' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout fsid = e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.38:3300,v1:172.21.15.38:6789] 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:33:02.129 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:33:02.130 INFO:teuthology.orchestra.run.smithi038.stdout:Generating new minimal ceph.conf... 2024-04-26T04:33:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:01 smithi038 ceph-mon[26083]: from='client.? 172.21.15.38:0/1126651264' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:33:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:01 smithi038 ceph-mon[26083]: from='client.? 172.21.15.38:0/1126651264' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-26T04:33:03.157 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:02 smithi038 ceph-mon[26083]: from='client.? 172.21.15.38:0/757988536' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:33:03.159 INFO:teuthology.orchestra.run.smithi038.stdout:Restarting the monitor... 2024-04-26T04:33:03.447 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:03 smithi038 systemd[1]: Stopping Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:33:03.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26079]: 2024-04-26T04:33:03.445+0000 7f8e4b5bf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:33:03.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26079]: 2024-04-26T04:33:03.445+0000 7f8e4b5bf700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-26T04:33:03.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:03 smithi038 podman[26329]: 2024-04-26 04:33:03.569111963 +0000 UTC m=+0.239786064 container died 30d1026b4fce06db7366917643aea8b02a5f33366c9d820c46f96eba8c709547 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c) 2024-04-26T04:33:04.080 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:03 smithi038 podman[26329]: 2024-04-26 04:33:03.895748154 +0000 UTC m=+0.566422252 container cleanup 30d1026b4fce06db7366917643aea8b02a5f33366c9d820c46f96eba8c709547 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T04:33:04.080 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:03 smithi038 bash[26329]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:33:04.330 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 podman[26343]: 2024-04-26 04:33:04.07988059 +0000 UTC m=+0.505312191 container remove 30d1026b4fce06db7366917643aea8b02a5f33366c9d820c46f96eba8c709547 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T04:33:04.598 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:33:04.598 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 systemd[1]: Stopped Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:33:04.598 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 systemd[1]: Starting Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:33:04.598 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 podman[26435]: 2024-04-26 04:33:04.450758911 +0000 UTC m=+0.018498122 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:33:04.598 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 podman[26435]: 2024-04-26 04:33:04.597424639 +0000 UTC m=+0.165163830 container create 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-26T04:33:04.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 podman[26435]: 2024-04-26 04:33:04.791366958 +0000 UTC m=+0.359106150 container init 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-26T04:33:04.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 podman[26435]: 2024-04-26 04:33:04.794443398 +0000 UTC m=+0.362182592 container start 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c) 2024-04-26T04:33:04.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:33:04.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: pidfile_write: ignore empty --pid-file 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: load: jerasure load: lrc 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: DB SUMMARY 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: DB Session ID: HB0BPPDTP74MKC0O5ZNA 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84229 ; 2024-04-26T04:33:04.876 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.env: 0x558319e7cb00 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.fs: Posix File System 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.info_log: 0x55831b332ee0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.statistics: (nil) 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.use_fsync: 0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:33:04.877 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.db_log_dir: 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:33:04.878 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.write_buffer_manager: 0x55831c0a2990 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.unordered_write: 0 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:33:04.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.row_cache: None 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.wal_filter: None 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:33:04.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:33:04.881 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:33:04.881 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:33:04.881 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:33:04.881 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:33:04.882 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:33:04.883 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:33:04.883 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:33:04.883 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:33:04.883 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_open_files: -1 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Compression algorithms supported: 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kZSTD supported: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:33:04.884 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.merge_operator: 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_filter: None 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55831b382f58) 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:33:04.885 INFO:journalctl@ceph.mon.a.smithi038.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: index_type: 0 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: data_block_index_type: 0 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: index_shortening: 1 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: hash_index_allow_collision: 1 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: checksum: 1 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: no_block_cache: 0 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache: 0x55831b3b5090 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache_options: 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: capacity : 536870912 2024-04-26T04:33:04.886 INFO:journalctl@ceph.mon.a.smithi038.stdout: num_shard_bits : 4 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: strict_capacity_limit : 0 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache_compressed: (nil) 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: persistent_cache: (nil) 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_size: 4096 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_size_deviation: 10 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_restart_interval: 16 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: index_block_restart_interval: 1 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: metadata_block_size: 4096 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: partition_filters: 0 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: use_delta_encoding: 1 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: whole_key_filtering: 1 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: verify_compression: 0 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:33:04.887 INFO:journalctl@ceph.mon.a.smithi038.stdout: format_version: 4 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout: enable_index_compression: 1 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_align: 0 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression: NoCompression 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.num_levels: 7 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:33:04.888 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:33:04.889 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:33:04.890 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:33:04.891 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:33:04.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:33:04.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.ttl: 2592000 2024-04-26T04:33:04.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:33:04.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:33:04.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:33:04.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:33:04.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714105984823492, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-26T04:33:04.894 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714105984824507, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80661, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78922, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73131, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714105984, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "553bf848-7d0a-42c2-9968-19ee9f0609ad", "db_session_id": "HB0BPPDTP74MKC0O5ZNA"}} 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714105984825344, "job": 1, "event": "recovery_finished"} 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55831b349180 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: DB pointer 0x55831b41e000 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** DB Stats ** 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:33:04.895 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: L0 2/0 80.53 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Sum 2/0 80.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:04.896 INFO:journalctl@ceph.mon.a.smithi038.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative compaction: 0.00 GB write, 11.08 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval compaction: 0.00 GB write, 11.08 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:04.897 INFO:journalctl@ceph.mon.a.smithi038.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: L0 2/0 80.53 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: Sum 2/0 80.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 102.8 0.00 0.00 1 0.001 0 0 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:33:04.898 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative compaction: 0.00 GB write, 11.03 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: starting mon.a rank 0 at public addrs [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] at bind addrs [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???) e1 preinit fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).mds e1 new map 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).mds e1 print_map 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: e1 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: legacy client fscid: -1 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout: No filesystems configured 2024-04-26T04:33:04.899 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:33:04.900 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:33:04.900 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:33:04.900 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:33:04.900 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-26T04:33:04.900 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 bash[26435]: 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a 2024-04-26T04:33:04.900 INFO:teuthology.orchestra.run.smithi038.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-26T04:33:05.127 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 systemd[1]: Started Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:33:05.127 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-26T04:33:05.127 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: monmap e1: 1 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:33:05.127 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: fsmap 2024-04-26T04:33:05.127 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: osdmap e1: 0 total, 0 up, 0 in 2024-04-26T04:33:05.127 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:04 smithi038 ceph-mon[26454]: mgrmap e1: no daemons active 2024-04-26T04:33:05.893 INFO:teuthology.orchestra.run.smithi038.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-26T04:33:05.893 INFO:teuthology.orchestra.run.smithi038.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:33:05.893 INFO:teuthology.orchestra.run.smithi038.stdout:Creating mgr... 2024-04-26T04:33:05.893 INFO:teuthology.orchestra.run.smithi038.stdout:Verifying port 9283 ... 2024-04-26T04:33:06.080 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y 2024-04-26T04:33:06.080 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Failed to reset failed state of unit ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-26T04:33:06.238 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-e22c1684-0385-11ef-bc93-c7b262605968.target.wants/ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-e22c1684-0385-11ef-bc93-c7b262605968@.service. 2024-04-26T04:33:06.393 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 systemd[1]: Starting Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:33:06.701 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:06 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4071494749' entity='client.admin' 2024-04-26T04:33:06.701 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 podman[26669]: 2024-04-26 04:33:06.379531341 +0000 UTC m=+0.018008383 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-26T04:33:06.701 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 podman[26669]: 2024-04-26 04:33:06.482075147 +0000 UTC m=+0.120552178 container create 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T04:33:06.701 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 podman[26669]: 2024-04-26 04:33:06.700866866 +0000 UTC m=+0.339343897 container init 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-26T04:33:06.799 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:33:06.799 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout disabled 2024-04-26T04:33:06.806 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:33:06.807 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout inactive 2024-04-26T04:33:06.807 INFO:teuthology.orchestra.run.smithi038.stdout:firewalld.service is not enabled 2024-04-26T04:33:06.807 INFO:teuthology.orchestra.run.smithi038.stdout:Not possible to enable service . firewalld.service is not available 2024-04-26T04:33:06.816 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:33:06.816 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout disabled 2024-04-26T04:33:06.825 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:33:06.825 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout inactive 2024-04-26T04:33:06.825 INFO:teuthology.orchestra.run.smithi038.stdout:firewalld.service is not enabled 2024-04-26T04:33:06.825 INFO:teuthology.orchestra.run.smithi038.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-26T04:33:06.825 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for mgr to start... 2024-04-26T04:33:06.825 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for mgr... 2024-04-26T04:33:06.953 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 podman[26669]: 2024-04-26 04:33:06.70360213 +0000 UTC m=+0.342079168 container start 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-26T04:33:06.953 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 bash[26669]: 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f 2024-04-26T04:33:06.953 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 systemd[1]: Started Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:33:06.953 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:06.856+0000 7fc5291f81c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:33:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:07 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:07.024+0000 7fc5291f81c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:07 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:07.129+0000 7fc5291f81c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:33:07.547 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:07 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3134044764' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:33:07.548 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:07 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:07.459+0000 7fc5291f81c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:33:07.894 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "fsid": "e22c1684-0385-11ef-bc93-c7b262605968", 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:07.897 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:33:07.898 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:33:07.902 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:33:07.903 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:32:59.902230+0000", 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:07.904 INFO:teuthology.orchestra.run.smithi038.stdout:mgr not available, waiting (1/15)... 2024-04-26T04:33:08.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:07 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:07.545+0000 7fc5291f81c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:33:09.105 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:08.763+0000 7fc5291f81c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:33:09.105 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:08.931+0000 7fc5291f81c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:09.105 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:09.103+0000 7fc5291f81c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:33:09.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:09.372+0000 7fc5291f81c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:33:09.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:09.465+0000 7fc5291f81c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:33:09.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:09.560+0000 7fc5291f81c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:33:10.032 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:09.845+0000 7fc5291f81c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:10 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2019175235' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:33:10.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:10.333+0000 7fc5291f81c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "fsid": "e22c1684-0385-11ef-bc93-c7b262605968", 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:33:11.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:11.024 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:33:11.025 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:11.026 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:33:11.027 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:11.028 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:32:59.902230+0000", 2024-04-26T04:33:11.028 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:11.028 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:11.028 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:33:11.028 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:11.028 INFO:teuthology.orchestra.run.smithi038.stdout:mgr not available, waiting (2/15)... 2024-04-26T04:33:11.168 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:10.886+0000 7fc5291f81c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:33:11.168 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:10.983+0000 7fc5291f81c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:33:11.168 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:11 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:11.165+0000 7fc5291f81c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:33:11.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:11 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:11.465+0000 7fc5291f81c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:33:12.109 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:11 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:11.729+0000 7fc5291f81c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:33:12.110 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:11 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:11.972+0000 7fc5291f81c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:33:12.110 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:12.107+0000 7fc5291f81c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:33:12.411 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:12.202+0000 7fc5291f81c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:33:12.411 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:12.319+0000 7fc5291f81c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:12.411 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:12.409+0000 7fc5291f81c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:33:13.446 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:13.240+0000 7fc5291f81c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:13.439+0000 7fc5291f81c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: Activating manager daemon y 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: mgrmap e2: y(active, starting, since 0.00311593s) 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: Manager daemon y is now available 2024-04-26T04:33:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:33:13.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:33:13.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' 2024-04-26T04:33:13.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' 2024-04-26T04:33:13.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:13 smithi038 ceph-mon[26454]: from='mgr.14100 172.21.15.38:0/2269145985' entity='mgr.y' 2024-04-26T04:33:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:15 smithi038 ceph-mon[26454]: mgrmap e3: y(active, since 1.01164s) 2024-04-26T04:33:16.859 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:16 smithi038 ceph-mon[26454]: mgrmap e4: y(active, since 2s) 2024-04-26T04:33:16.859 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:16 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/751558336' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-26T04:33:22.792 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:22.792 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:22.792 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "fsid": "e22c1684-0385-11ef-bc93-c7b262605968", 2024-04-26T04:33:22.792 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "health": { 2024-04-26T04:33:22.792 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 0 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:22.793 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "a" 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.794 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-26T04:33:22.795 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-26T04:33:22.796 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.797 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "restful" 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ], 2024-04-26T04:33:22.798 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "modified": "2024-04-26T04:32:59.902230+0000", 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout }, 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-26T04:33:22.799 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:22.800 INFO:teuthology.orchestra.run.smithi038.stdout:mgr is available 2024-04-26T04:33:24.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:23 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/425690827' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-26T04:33:24.434 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout [global] 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout fsid = e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-26T04:33:24.436 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 2024-04-26T04:33:24.437 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout [osd] 2024-04-26T04:33:24.437 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-26T04:33:24.437 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-26T04:33:24.437 INFO:teuthology.orchestra.run.smithi038.stdout:Enabling cephadm module... 2024-04-26T04:33:25.420 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/355973704' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-26T04:33:25.421 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setuser ceph since I am not root 2024-04-26T04:33:25.421 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setgroup ceph since I am not root 2024-04-26T04:33:25.421 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:25.277+0000 7fbc180121c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:33:25.679 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:25.445+0000 7fbc180121c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:25.679 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:25.548+0000 7fbc180121c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:33:25.930 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:25.874+0000 7fbc180121c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:33:26.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:25.964+0000 7fbc180121c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:33:26.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:26 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/355973704' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-26T04:33:26.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:26 smithi038 ceph-mon[26454]: mgrmap e5: y(active, since 11s) 2024-04-26T04:33:27.022 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for the mgr to restart... 2024-04-26T04:33:27.023 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for mgr epoch 5... 2024-04-26T04:33:27.354 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2457405175' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:33:27.354 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:27.200+0000 7fbc180121c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:33:27.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:27.375+0000 7fbc180121c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:27.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:27.571+0000 7fbc180121c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:33:28.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:27.841+0000 7fbc180121c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:33:28.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:27.934+0000 7fbc180121c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:33:28.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:28 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:28.030+0000 7fbc180121c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:33:28.727 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:28 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:28.315+0000 7fbc180121c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:29.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:28 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:28.795+0000 7fbc180121c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:33:29.614 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:29 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:29.335+0000 7fbc180121c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:33:29.614 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:29 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:29.431+0000 7fbc180121c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:33:29.614 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:29 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:29.613+0000 7fbc180121c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:33:30.181 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:29 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:29.915+0000 7fbc180121c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:33:30.181 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:30.180+0000 7fbc180121c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:33:30.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:30.425+0000 7fbc180121c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:33:30.865 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:30.560+0000 7fbc180121c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:33:30.866 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:30.655+0000 7fbc180121c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:33:30.866 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:30.773+0000 7fbc180121c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:30.866 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:30.864+0000 7fbc180121c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:31.684+0000 7fbc180121c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:31.880+0000 7fbc180121c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: Active manager daemon y restarted 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: Activating manager daemon y 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: osdmap e2: 0 total, 0 up, 0 in 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: mgrmap e6: y(active, starting, since 0.00532344s) 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:33:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: Manager daemon y is now available 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:33:31.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:31 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:33:33.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:32 smithi038 ceph-mon[26454]: Found migration_current of "None". Setting to last migration. 2024-04-26T04:33:33.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:32 smithi038 ceph-mon[26454]: mgrmap e7: y(active, since 1.00869s) 2024-04-26T04:33:33.469 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:33.469 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-26T04:33:33.469 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:33:33.469 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:33.469 INFO:teuthology.orchestra.run.smithi038.stdout:mgr epoch 5 is available 2024-04-26T04:33:33.469 INFO:teuthology.orchestra.run.smithi038.stdout:Setting orchestrator backend to cephadm... 2024-04-26T04:33:34.141 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:33 smithi038 ceph-mon[26454]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-26T04:33:34.141 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:33 smithi038 ceph-mon[26454]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-26T04:33:34.141 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:33 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:34.142 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:33 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: [26/Apr/2024:04:33:33] ENGINE Bus STARTING 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: [26/Apr/2024:04:33:33] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: [26/Apr/2024:04:33:33] ENGINE Bus STARTED 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: mgrmap e8: y(active, since 2s) 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:35.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:34 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:35.856 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-26T04:33:35.857 INFO:teuthology.orchestra.run.smithi038.stdout:Generating ssh key... 2024-04-26T04:33:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-mon[26454]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: Generating public/private rsa key pair. 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: Your identification has been saved in /tmp/tmpahwkgag9/key. 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: Your public key has been saved in /tmp/tmpahwkgag9/key.pub. 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: The key fingerprint is: 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: SHA256:ViRQB+UgPRGl7E9ZNlhf3HkVx74u9fEymCFrspGE1Lo ceph-e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: The key's randomart image is: 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: +---[RSA 3072]----+ 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | o+B*= . +O| 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | +oB o . o*| 2024-04-26T04:33:36.960 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | . +.+ + ...| 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | . + . + . .| 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | o S o .| 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | + +. . + | 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | E o .o +o +| 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | ..o o.o.o| 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: | .+ .o | 2024-04-26T04:33:36.961 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: +----[SHA256]-----+ 2024-04-26T04:33:37.868 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:37 smithi038 ceph-mon[26454]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:37.869 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:37 smithi038 ceph-mon[26454]: Generating ssh key... 2024-04-26T04:33:37.869 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:37 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:37.869 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:37 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:38.420 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXJO4SQ1p7YmezLEaVawCz/RKYGvhVqpp2zzU3IU3VxQvGL+I506ivXXxN9f+shbc0pVtxTBG9edaFY3WZjYu2M65/WinjGoGODUKstK63P69IpXnQM2tHO0eI3TFx+zQeanS/WnrH8HO1DCUpe3rpmZve9RbJUqh1NJH93+AV/xmluYzV0YuPZVvnaoBLxLmJ6Pb3LtB+M0+QH7LwLcezvVQHZMdpYk+KRAj1sqtAuoW0R9bwLCZF4+l/UZctZ7z51123VbYfoJBjCJNOcF2boB4zZW3wZ2xZUfjWFkuOLaYWY3h6PJFY5yMNXdr2hbYnNGPRIl2rwhy9bMaAZ/xsm7jj+A/1FH4s2cFyqDumKg4Ehi3USYNFGQvqXw0EggrVtuYI4CO+I4CRA5kzxTiA/rCEf4NmrbWZa/b2XMdYiI6rhbohQB/o4jzJAO32vJR5Ry4XfIOVkYoMWc72zwiWP0tIzRx9XOgMSkSSa9BbQ53UjVWKyrtnPL2Q1haKyhU= ceph-e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:33:38.420 INFO:teuthology.orchestra.run.smithi038.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-26T04:33:38.420 INFO:teuthology.orchestra.run.smithi038.stdout:Adding key to root@localhost authorized_keys... 2024-04-26T04:33:38.420 INFO:teuthology.orchestra.run.smithi038.stdout:Adding host smithi038... 2024-04-26T04:33:38.861 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:38 smithi038 ceph-mon[26454]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:39 smithi038 ceph-mon[26454]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi038", "addr": "172.21.15.38", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:40 smithi038 ceph-mon[26454]: Deploying cephadm binary to smithi038 2024-04-26T04:33:41.535 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout Added host 'smithi038' with addr '172.21.15.38' 2024-04-26T04:33:41.536 INFO:teuthology.orchestra.run.smithi038.stdout:Deploying unmanaged mon service... 2024-04-26T04:33:42.222 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:41 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:42.222 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:41 smithi038 ceph-mon[26454]: Added host smithi038 2024-04-26T04:33:42.222 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:41 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:42.736 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-26T04:33:42.737 INFO:teuthology.orchestra.run.smithi038.stdout:Deploying unmanaged mgr service... 2024-04-26T04:33:43.393 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:43 smithi038 ceph-mon[26454]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:43.393 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:43 smithi038 ceph-mon[26454]: Saving service mon spec with placement count:5 2024-04-26T04:33:43.394 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:43 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:44.470 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:44 smithi038 ceph-mon[26454]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:33:44.471 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:44 smithi038 ceph-mon[26454]: Saving service mgr spec with placement count:2 2024-04-26T04:33:44.471 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:44 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:44.471 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:44 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:44.774 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-26T04:33:46.139 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:45 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:46.139 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:45 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2133454639' entity='client.admin' 2024-04-26T04:33:47.649 INFO:teuthology.orchestra.run.smithi038.stdout:Enabling the dashboard module... 2024-04-26T04:33:48.049 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:47 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:48.049 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:47 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1296481841' entity='client.admin' 2024-04-26T04:33:48.049 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:47 smithi038 ceph-mon[26454]: from='mgr.14118 172.21.15.38:0/130112618' entity='mgr.y' 2024-04-26T04:33:49.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:48 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1088631037' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-26T04:33:49.226 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setuser ceph since I am not root 2024-04-26T04:33:49.226 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setgroup ceph since I am not root 2024-04-26T04:33:49.226 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:49.152+0000 7f72261341c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:33:49.560 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:49.318+0000 7f72261341c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:49.560 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:49.422+0000 7f72261341c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:33:49.831 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:49.739+0000 7f72261341c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:33:49.831 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:49.829+0000 7f72261341c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:33:50.084 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1088631037' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-26T04:33:50.084 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:49 smithi038 ceph-mon[26454]: mgrmap e9: y(active, since 17s) 2024-04-26T04:33:51.109 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:50 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/174452508' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-26T04:33:51.109 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:51.047+0000 7f72261341c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for the mgr to restart... 2024-04-26T04:33:51.112 INFO:teuthology.orchestra.run.smithi038.stdout:Waiting for mgr epoch 9... 2024-04-26T04:33:51.379 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:51.226+0000 7f72261341c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:51.678 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:51.403+0000 7f72261341c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:33:51.678 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:51.676+0000 7f72261341c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:33:51.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:51.770+0000 7f72261341c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:33:51.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:51.866+0000 7f72261341c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:33:52.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:52.151+0000 7f72261341c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:33:52.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:52.627+0000 7f72261341c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:33:53.449 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:53.169+0000 7f72261341c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:33:53.449 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:53.265+0000 7f72261341c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:33:53.449 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:53.447+0000 7f72261341c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:33:54.015 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:53.749+0000 7f72261341c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:33:54.015 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:54.014+0000 7f72261341c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:33:54.396 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:54.258+0000 7f72261341c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:33:54.396 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:54.395+0000 7f72261341c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:33:54.704 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:54.492+0000 7f72261341c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:33:54.705 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:54.612+0000 7f72261341c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:33:54.705 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:54.703+0000 7f72261341c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:33:55.791 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:55.538+0000 7f72261341c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:33:55.791 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:33:55.737+0000 7f72261341c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: Active manager daemon y restarted 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: Activating manager daemon y 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: osdmap e3: 0 total, 0 up, 0 in 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: mgrmap e10: y(active, starting, since 0.0050068s) 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:33:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:33:56.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: Manager daemon y is now available 2024-04-26T04:33:56.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:56.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:33:56.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:33:56.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:55 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:33:57.489 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout { 2024-04-26T04:33:57.489 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-26T04:33:57.489 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-26T04:33:57.489 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout } 2024-04-26T04:33:57.489 INFO:teuthology.orchestra.run.smithi038.stdout:mgr epoch 9 is available 2024-04-26T04:33:57.489 INFO:teuthology.orchestra.run.smithi038.stdout:Generating a dashboard self-signed certificate... 2024-04-26T04:33:57.749 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:57 smithi038 ceph-mon[26454]: mgrmap e11: y(active, since 1.00583s) 2024-04-26T04:33:57.749 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:57 smithi038 ceph-mon[26454]: [26/Apr/2024:04:33:56] ENGINE Bus STARTING 2024-04-26T04:33:57.749 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:57 smithi038 ceph-mon[26454]: [26/Apr/2024:04:33:56] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:33:57.749 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:57 smithi038 ceph-mon[26454]: [26/Apr/2024:04:33:56] ENGINE Bus STARTED 2024-04-26T04:33:57.749 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:57 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:33:57.749 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:57 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:33:58.745 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-26T04:33:58.745 INFO:teuthology.orchestra.run.smithi038.stdout:Creating initial admin user... 2024-04-26T04:33:59.120 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:33:59.121 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:58 smithi038 ceph-mon[26454]: mgrmap e12: y(active, since 2s) 2024-04-26T04:33:59.121 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:33:59.121 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:59 smithi038 ceph-mon[26454]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:33:59 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:01.084 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$x5wuRp.pxaIuRFZZnC0bEuG2NwU6N2/TP2kjPbJvBep.PtUMY5fsG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714106039, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-26T04:34:01.084 INFO:teuthology.orchestra.run.smithi038.stdout:Fetching dashboard port number... 2024-04-26T04:34:01.275 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:00 smithi038 ceph-mon[26454]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:01 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/281775438' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-26T04:34:02.240 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stdout 8443 2024-04-26T04:34:02.251 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-26T04:34:02.251 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout disabled 2024-04-26T04:34:02.260 INFO:teuthology.orchestra.run.smithi038.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-26T04:34:02.260 INFO:teuthology.orchestra.run.smithi038.stdout:systemctl: stdout inactive 2024-04-26T04:34:02.260 INFO:teuthology.orchestra.run.smithi038.stdout:firewalld.service is not enabled 2024-04-26T04:34:02.260 INFO:teuthology.orchestra.run.smithi038.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-26T04:34:02.261 INFO:teuthology.orchestra.run.smithi038.stdout:Ceph Dashboard is now available at: 2024-04-26T04:34:02.261 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:02.261 INFO:teuthology.orchestra.run.smithi038.stdout: URL: https://smithi038:8443/ 2024-04-26T04:34:02.262 INFO:teuthology.orchestra.run.smithi038.stdout: User: admin 2024-04-26T04:34:02.262 INFO:teuthology.orchestra.run.smithi038.stdout: Password: wi2craorp5 2024-04-26T04:34:02.262 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:02.262 INFO:teuthology.orchestra.run.smithi038.stdout:Saving cluster configuration to /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config directory 2024-04-26T04:34:02.262 INFO:teuthology.orchestra.run.smithi038.stdout:Enabling autotune for osd_memory_target 2024-04-26T04:34:03.016 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:03 smithi038 ceph-mon[26454]: mgrmap e13: y(active, since 6s) 2024-04-26T04:34:05.134 INFO:teuthology.orchestra.run.smithi038.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout:Or, if you are only running a single cluster on this host: 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.135 INFO:teuthology.orchestra.run.smithi038.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout: ceph telemetry on 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout:For more information see: 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:05.136 INFO:teuthology.orchestra.run.smithi038.stdout:Bootstrap complete. 2024-04-26T04:34:05.174 INFO:tasks.cephadm:Fetching config... 2024-04-26T04:34:05.174 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:05.174 DEBUG:teuthology.orchestra.run.smithi038:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-26T04:34:05.192 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-26T04:34:05.192 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:05.193 DEBUG:teuthology.orchestra.run.smithi038:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-26T04:34:05.249 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-26T04:34:05.250 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:05.250 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd if=/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-26T04:34:05.317 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-26T04:34:05.317 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:05.317 DEBUG:teuthology.orchestra.run.smithi038:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-26T04:34:05.373 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-26T04:34:05.373 DEBUG:teuthology.orchestra.run.smithi038:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXJO4SQ1p7YmezLEaVawCz/RKYGvhVqpp2zzU3IU3VxQvGL+I506ivXXxN9f+shbc0pVtxTBG9edaFY3WZjYu2M65/WinjGoGODUKstK63P69IpXnQM2tHO0eI3TFx+zQeanS/WnrH8HO1DCUpe3rpmZve9RbJUqh1NJH93+AV/xmluYzV0YuPZVvnaoBLxLmJ6Pb3LtB+M0+QH7LwLcezvVQHZMdpYk+KRAj1sqtAuoW0R9bwLCZF4+l/UZctZ7z51123VbYfoJBjCJNOcF2boB4zZW3wZ2xZUfjWFkuOLaYWY3h6PJFY5yMNXdr2hbYnNGPRIl2rwhy9bMaAZ/xsm7jj+A/1FH4s2cFyqDumKg4Ehi3USYNFGQvqXw0EggrVtuYI4CO+I4CRA5kzxTiA/rCEf4NmrbWZa/b2XMdYiI6rhbohQB/o4jzJAO32vJR5Ry4XfIOVkYoMWc72zwiWP0tIzRx9XOgMSkSSa9BbQ53UjVWKyrtnPL2Q1haKyhU= ceph-e22c1684-0385-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:34:05.460 INFO:teuthology.orchestra.run.smithi038.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXJO4SQ1p7YmezLEaVawCz/RKYGvhVqpp2zzU3IU3VxQvGL+I506ivXXxN9f+shbc0pVtxTBG9edaFY3WZjYu2M65/WinjGoGODUKstK63P69IpXnQM2tHO0eI3TFx+zQeanS/WnrH8HO1DCUpe3rpmZve9RbJUqh1NJH93+AV/xmluYzV0YuPZVvnaoBLxLmJ6Pb3LtB+M0+QH7LwLcezvVQHZMdpYk+KRAj1sqtAuoW0R9bwLCZF4+l/UZctZ7z51123VbYfoJBjCJNOcF2boB4zZW3wZ2xZUfjWFkuOLaYWY3h6PJFY5yMNXdr2hbYnNGPRIl2rwhy9bMaAZ/xsm7jj+A/1FH4s2cFyqDumKg4Ehi3USYNFGQvqXw0EggrVtuYI4CO+I4CRA5kzxTiA/rCEf4NmrbWZa/b2XMdYiI6rhbohQB/o4jzJAO32vJR5Ry4XfIOVkYoMWc72zwiWP0tIzRx9XOgMSkSSa9BbQ53UjVWKyrtnPL2Q1haKyhU= ceph-e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:34:05.474 DEBUG:teuthology.orchestra.run.smithi055:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXJO4SQ1p7YmezLEaVawCz/RKYGvhVqpp2zzU3IU3VxQvGL+I506ivXXxN9f+shbc0pVtxTBG9edaFY3WZjYu2M65/WinjGoGODUKstK63P69IpXnQM2tHO0eI3TFx+zQeanS/WnrH8HO1DCUpe3rpmZve9RbJUqh1NJH93+AV/xmluYzV0YuPZVvnaoBLxLmJ6Pb3LtB+M0+QH7LwLcezvVQHZMdpYk+KRAj1sqtAuoW0R9bwLCZF4+l/UZctZ7z51123VbYfoJBjCJNOcF2boB4zZW3wZ2xZUfjWFkuOLaYWY3h6PJFY5yMNXdr2hbYnNGPRIl2rwhy9bMaAZ/xsm7jj+A/1FH4s2cFyqDumKg4Ehi3USYNFGQvqXw0EggrVtuYI4CO+I4CRA5kzxTiA/rCEf4NmrbWZa/b2XMdYiI6rhbohQB/o4jzJAO32vJR5Ry4XfIOVkYoMWc72zwiWP0tIzRx9XOgMSkSSa9BbQ53UjVWKyrtnPL2Q1haKyhU= ceph-e22c1684-0385-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-26T04:34:05.515 INFO:teuthology.orchestra.run.smithi055.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXJO4SQ1p7YmezLEaVawCz/RKYGvhVqpp2zzU3IU3VxQvGL+I506ivXXxN9f+shbc0pVtxTBG9edaFY3WZjYu2M65/WinjGoGODUKstK63P69IpXnQM2tHO0eI3TFx+zQeanS/WnrH8HO1DCUpe3rpmZve9RbJUqh1NJH93+AV/xmluYzV0YuPZVvnaoBLxLmJ6Pb3LtB+M0+QH7LwLcezvVQHZMdpYk+KRAj1sqtAuoW0R9bwLCZF4+l/UZctZ7z51123VbYfoJBjCJNOcF2boB4zZW3wZ2xZUfjWFkuOLaYWY3h6PJFY5yMNXdr2hbYnNGPRIl2rwhy9bMaAZ/xsm7jj+A/1FH4s2cFyqDumKg4Ehi3USYNFGQvqXw0EggrVtuYI4CO+I4CRA5kzxTiA/rCEf4NmrbWZa/b2XMdYiI6rhbohQB/o4jzJAO32vJR5Ry4XfIOVkYoMWc72zwiWP0tIzRx9XOgMSkSSa9BbQ53UjVWKyrtnPL2Q1haKyhU= ceph-e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:34:05.528 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-26T04:34:05.646 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:05 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3862431058' entity='client.admin' 2024-04-26T04:34:05.796 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:08.341 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:08.342 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:08.342 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi038", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:34:08.342 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:08.342 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:08.342 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:09.876 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-26T04:34:09.876 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-26T04:34:10.136 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:10 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4163564697' entity='client.admin' 2024-04-26T04:34:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:12.397 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi055 2024-04-26T04:34:12.397 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:34:12.397 DEBUG:teuthology.orchestra.run.smithi055:> dd of=/etc/ceph/ceph.conf 2024-04-26T04:34:12.416 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:34:12.416 DEBUG:teuthology.orchestra.run.smithi055:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:12.474 INFO:tasks.cephadm:Adding host smithi055 to orchestrator... 2024-04-26T04:34:12.474 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch host add smithi055 2024-04-26T04:34:12.734 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:12.847 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:12.847 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:12.848 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:13 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:13 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:34:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:15 smithi038 ceph-mon[26454]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi055", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:15 smithi038 ceph-mon[26454]: Deploying cephadm binary to smithi055 2024-04-26T04:34:16.312 INFO:teuthology.orchestra.run.smithi038.stdout:Added host 'smithi055' with addr '172.21.15.55' 2024-04-26T04:34:16.819 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-26T04:34:17.076 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:17.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:16 smithi038 ceph-mon[26454]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:17.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:17.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:17.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:18.000 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:17 smithi038 ceph-mon[26454]: Added host smithi055 2024-04-26T04:34:18.000 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:18.000 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:18.679 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:34:18.679 INFO:teuthology.orchestra.run.smithi038.stdout:[{"addr": "172.21.15.38", "hostname": "smithi038", "labels": [], "status": ""}, {"addr": "172.21.15.55", "hostname": "smithi055", "labels": [], "status": ""}] 2024-04-26T04:34:19.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:18 smithi038 ceph-mon[26454]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:19.266 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-26T04:34:19.266 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-26T04:34:19.524 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:20.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:20 smithi038 ceph-mon[26454]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:34:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:21.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:21.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:21.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi055", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:34:21.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:21.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:21 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:22.025 INFO:teuthology.orchestra.run.smithi038.stderr:adjusted tunables profile to default 2024-04-26T04:34:22.328 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:22 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:34:22.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:22 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:22.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:22 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/901127657' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:34:22.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:22 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:22.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:22 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:22.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:22 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:22.628 INFO:tasks.cephadm:Adding mon.a on smithi038 2024-04-26T04:34:22.628 INFO:tasks.cephadm:Adding mon.c on smithi038 2024-04-26T04:34:22.628 INFO:tasks.cephadm:Adding mon.b on smithi055 2024-04-26T04:34:22.628 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi038:172.21.15.38=a;smithi038:[v2:172.21.15.38:3301,v1:172.21.15.38:6790]=c;smithi055:172.21.15.55=b' 2024-04-26T04:34:22.890 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:23 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:23 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:34:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:23 smithi038 ceph-mon[26454]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:23 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/901127657' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:34:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:23 smithi038 ceph-mon[26454]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:24.648 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled mon update... 2024-04-26T04:34:25.220 DEBUG:teuthology.orchestra.run.smithi038:mon.c> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.c.service 2024-04-26T04:34:25.223 DEBUG:teuthology.orchestra.run.smithi055:mon.b> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.b.service 2024-04-26T04:34:25.225 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:34:25.225 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:25 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:26 smithi038 ceph-mon[26454]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi038:172.21.15.38=a;smithi038:[v2:172.21.15.38:3301,v1:172.21.15.38:6790]=c;smithi055:172.21.15.55=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:26 smithi038 ceph-mon[26454]: Saving service mon spec with placement smithi038:172.21.15.38=a;smithi038:[v2:172.21.15.38:3301,v1:172.21.15.38:6790]=c;smithi055:172.21.15.55=b;count:3 2024-04-26T04:34:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:26 smithi038 ceph-mon[26454]: Deploying daemon mon.b on smithi055 2024-04-26T04:34:27.252 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:34:27.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:27 smithi038 ceph-mon[26454]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:28.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:28 smithi055 systemd[1]: Starting Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:34:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:29 smithi055 podman[30291]: 2024-04-26 04:34:29.305022501 +0000 UTC m=+0.109055668 container create e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T04:34:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:29 smithi055 podman[30291]: 2024-04-26 04:34:29.214788327 +0000 UTC m=+0.018821506 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:34:29.374 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:34:29.374 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":1,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","modified":"2024-04-26T04:32:57.061143Z","created":"2024-04-26T04:32:57.061143Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:3300","nonce":0},{"type":"v1","addr":"172.21.15.38:6789","nonce":0}]},"addr":"172.21.15.38:6789/0","public_addr":"172.21.15.38:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-26T04:34:29.376 INFO:teuthology.orchestra.run.smithi055.stderr:dumped monmap epoch 1 2024-04-26T04:34:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:29 smithi038 ceph-mon[26454]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:30.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 podman[30291]: 2024-04-26 04:34:30.027645495 +0000 UTC m=+0.831678655 container init e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T04:34:30.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 podman[30291]: 2024-04-26 04:34:30.031202203 +0000 UTC m=+0.835235363 container start e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-26T04:34:30.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:34:30.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable), process ceph-mon, pid 2 2024-04-26T04:34:30.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: pidfile_write: ignore empty --pid-file 2024-04-26T04:34:30.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: load: jerasure load: lrc 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: RocksDB version: 6.15.5 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Compile date Apr 24 2024 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: DB SUMMARY 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: DB Session ID: 20CO62NQ8WF5TZHMQ9EE 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:34:30.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.env: 0x55ac7980cb00 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.fs: Posix File System 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.info_log: 0x55ac7c00cee0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.statistics: (nil) 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.use_fsync: 0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:34:30.359 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.db_log_dir: 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:34:30.360 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.write_buffer_manager: 0x55ac7cd7c990 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.unordered_write: 0 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:34:30.361 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:34:30.362 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.row_cache: None 2024-04-26T04:34:30.362 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.wal_filter: None 2024-04-26T04:34:30.362 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:34:30.363 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:34:30.363 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.preserve_deletes: 0 2024-04-26T04:34:30.363 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:34:30.363 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:34:30.363 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:34:30.364 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_open_files: -1 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:34:30.365 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Compression algorithms supported: 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kZSTD supported: 0 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.merge_operator: 2024-04-26T04:34:30.366 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_filter: None 2024-04-26T04:34:30.367 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ac7c05cf58) 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: index_type: 0 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: data_block_index_type: 0 2024-04-26T04:34:30.370 INFO:journalctl@ceph.mon.b.smithi055.stdout: index_shortening: 1 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: hash_index_allow_collision: 1 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: checksum: 1 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: no_block_cache: 0 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache: 0x55ac7c08f090 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache_options: 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: capacity : 536870912 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: num_shard_bits : 4 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: strict_capacity_limit : 0 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache_compressed: (nil) 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: persistent_cache: (nil) 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_size: 4096 2024-04-26T04:34:30.371 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_size_deviation: 10 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_restart_interval: 16 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: index_block_restart_interval: 1 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: metadata_block_size: 4096 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: partition_filters: 0 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: use_delta_encoding: 1 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: whole_key_filtering: 1 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: verify_compression: 0 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: format_version: 4 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: enable_index_compression: 1 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_align: 0 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:34:30.372 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression: NoCompression 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.num_levels: 7 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:34:30.373 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:34:30.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.arena_block_size: 4194304 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-26T04:34:30.375 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:34:30.376 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.ttl: 2592000 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:34:30.377 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106070061049, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106070061482, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714106070, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "681a96e0-f04e-4d79-a00f-b550bb1737e8", "db_session_id": "20CO62NQ8WF5TZHMQ9EE"}} 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-26T04:34:30.378 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106070062070, "job": 1, "event": "recovery_finished"} 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55ac7c023180 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: DB pointer 0x55ac7c0f8000 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** DB Stats ** 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:34:30.379 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** Compaction Stats [default] ** 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** Compaction Stats [default] ** 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:34:30.380 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** Compaction Stats [default] ** 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-26T04:34:30.381 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** Compaction Stats [default] ** 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:34:30.382 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: using public_addr v2:172.21.15.55:0/0 -> [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: starting mon.b rank -1 at public addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] at bind addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(???) e0 preinit fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).mds e1 new map 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: e1 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: legacy client fscid: -1 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout: No filesystems configured 2024-04-26T04:34:30.383 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/901127657' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:30.384 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/901127657' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi038:172.21.15.38=a;smithi038:[v2:172.21.15.38:3301,v1:172.21.15.38:6790]=c;smithi055:172.21.15.55=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: Saving service mon spec with placement smithi038:172.21.15.38=a;smithi038:[v2:172.21.15.38:3301,v1:172.21.15.38:6790]=c;smithi055:172.21.15.55=b;count:3 2024-04-26T04:34:30.385 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: Deploying daemon mon.b on smithi055 2024-04-26T04:34:30.386 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:30.386 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:30.386 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/3830832788' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:30.386 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 ceph-mon[30328]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-26T04:34:30.386 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 bash[30291]: e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 2024-04-26T04:34:30.386 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:30 smithi055 systemd[1]: Started Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:34:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/3830832788' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:30.972 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:34:30.973 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:34:31.242 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: mon.a calling monitor election 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: mon.b calling monitor election 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: monmap e2: 2 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0],b=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: fsmap 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: mgrmap e13: y(active, since 39s) 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: overall HEALTH_OK 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:35.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:35.560 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: mon.a calling monitor election 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: mon.b calling monitor election 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: monmap e2: 2 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0],b=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: fsmap 2024-04-26T04:34:35.561 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: mgrmap e13: y(active, since 39s) 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: overall HEALTH_OK 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:35.562 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:36.071 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:34:36.072 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":2,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","modified":"2024-04-26T04:34:30.086903Z","created":"2024-04-26T04:32:57.061143Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:3300","nonce":0},{"type":"v1","addr":"172.21.15.38:6789","nonce":0}]},"addr":"172.21.15.38:6789/0","public_addr":"172.21.15.38:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-26T04:34:36.074 INFO:teuthology.orchestra.run.smithi055.stderr:dumped monmap epoch 2 2024-04-26T04:34:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:36 smithi055 ceph-mon[30328]: Deploying daemon mon.c on smithi038 2024-04-26T04:34:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:36 smithi055 ceph-mon[30328]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:36 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/859000565' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:36.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:36 smithi038 ceph-mon[26454]: Deploying daemon mon.c on smithi038 2024-04-26T04:34:36.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:36 smithi038 ceph-mon[26454]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:36.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:36 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/859000565' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:36.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:37.767 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-26T04:34:37.767 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-26T04:34:38.044 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:34:38.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:38 smithi038 systemd[1]: Started Ceph mon.c for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: mon.a calling monitor election 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: mon.b calling monitor election 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: mon.c calling monitor election 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: monmap e3: 3 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0],b=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],c=[v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: fsmap 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: mgrmap e13: y(active, since 47s) 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: overall HEALTH_OK 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: mon.a calling monitor election 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: mon.b calling monitor election 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: mon.c calling monitor election 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: monmap e3: 3 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0],b=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],c=[v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: fsmap 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: mgrmap e13: y(active, since 47s) 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: overall HEALTH_OK 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:34:43.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: mon.a calling monitor election 2024-04-26T04:34:43.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: mon.b calling monitor election 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: mon.c calling monitor election 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: monmap e3: 3 mons at {a=[v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0],b=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],c=[v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: fsmap 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: osdmap e4: 0 total, 0 up, 0 in 2024-04-26T04:34:43.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: mgrmap e13: y(active, since 47s) 2024-04-26T04:34:43.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: overall HEALTH_OK 2024-04-26T04:34:43.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:43.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: 2024-04-26T04:34:44.199 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":3,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","modified":"2024-04-26T04:34:38.393611Z","created":"2024-04-26T04:32:57.061143Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:3300","nonce":0},{"type":"v1","addr":"172.21.15.38:6789","nonce":0}]},"addr":"172.21.15.38:6789/0","public_addr":"172.21.15.38:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:3301","nonce":0},{"type":"v1","addr":"172.21.15.38:6790","nonce":0}]},"addr":"172.21.15.38:6790/0","public_addr":"172.21.15.38:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-26T04:34:44.201 INFO:teuthology.orchestra.run.smithi055.stderr:dumped monmap epoch 3 2024-04-26T04:34:44.828 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:44 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:44.828 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:44 smithi055 ceph-mon[30328]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:44.828 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:44 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/404894415' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:44.828 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:44 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:44.855 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-26T04:34:44.855 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-26T04:34:44.879 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:44.879 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[36743]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:44.879 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/404894415' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:44.879 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:44.879 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:44.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[26454]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:44.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/404894415' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-26T04:34:44.880 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:44 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:34:45.112 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:34:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:34:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:46 smithi055 ceph-mon[30328]: Reconfiguring daemon mon.a on smithi038 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:34:47.199 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[26454]: Reconfiguring daemon mon.a on smithi038 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: Reconfiguring mon.a (unknown last config time)... 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:47.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:46 smithi038 ceph-mon[36743]: Reconfiguring daemon mon.a on smithi038 2024-04-26T04:34:47.868 INFO:teuthology.orchestra.run.smithi038.stdout:# minimal ceph.conf for e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:34:47.868 INFO:teuthology.orchestra.run.smithi038.stdout:[global] 2024-04-26T04:34:47.868 INFO:teuthology.orchestra.run.smithi038.stdout: fsid = e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:34:47.868 INFO:teuthology.orchestra.run.smithi038.stdout: mon_host = [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] 2024-04-26T04:34:48.484 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-26T04:34:48.484 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:48.484 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: Reconfiguring daemon mon.c on smithi038 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1063395289' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:48.552 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: Reconfiguring daemon mon.c on smithi038 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:48.553 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:48 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1063395289' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:48.555 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:48.555 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:34:48.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:48.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:48.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:48.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: Reconfiguring daemon mon.c on smithi038 2024-04-26T04:34:48.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:48.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:48 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1063395289' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:48.624 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:34:48.624 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.conf 2024-04-26T04:34:48.657 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:34:48.657 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:34:48.728 INFO:tasks.cephadm:Adding mgr.y on smithi038 2024-04-26T04:34:48.728 INFO:tasks.cephadm:Adding mgr.x on smithi055 2024-04-26T04:34:48.728 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi038=y;smithi055=x' 2024-04-26T04:34:49.029 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:34:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:34:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:50.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:50.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: Reconfiguring daemon mon.b on smithi055 2024-04-26T04:34:50.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:50 smithi055 ceph-mon[30328]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: Reconfiguring daemon mon.b on smithi055 2024-04-26T04:34:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[26454]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: Reconfiguring daemon mon.b on smithi055 2024-04-26T04:34:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:50 smithi038 ceph-mon[36743]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:50.732 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled mgr update... 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi038=y;smithi055=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: Saving service mgr spec with placement smithi038=y;smithi055=x;count:2 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:52.032 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:52.058 DEBUG:teuthology.orchestra.run.smithi055:mgr.x> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service 2024-04-26T04:34:52.060 INFO:tasks.cephadm:Deploying OSDs... 2024-04-26T04:34:52.060 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:34:52.060 DEBUG:teuthology.orchestra.run.smithi038:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:34:52.081 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:34:52.082 DEBUG:teuthology.orchestra.run.smithi038:> stat /dev/vg_nvme/lv_1 2024-04-26T04:34:52.141 INFO:teuthology.orchestra.run.smithi038.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout:Access: 2024-04-26 04:34:37.822434275 +0000 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout:Modify: 2024-04-26 04:28:56.196676486 +0000 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout:Change: 2024-04-26 04:28:56.196676486 +0000 2024-04-26T04:34:52.142 INFO:teuthology.orchestra.run.smithi038.stdout: Birth: 2024-04-26 04:28:56.196676486 +0000 2024-04-26T04:34:52.142 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:34:52.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi038=y;smithi055=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:52.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: Saving service mgr spec with placement smithi038=y;smithi055=x;count:2 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi038=y;smithi055=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: Saving service mgr spec with placement smithi038=y;smithi055=x;count:2 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.208 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:52.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:52.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:52.211 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records in 2024-04-26T04:34:52.211 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records out 2024-04-26T04:34:52.211 INFO:teuthology.orchestra.run.smithi038.stderr:512 bytes copied, 0.000318492 s, 1.6 MB/s 2024-04-26T04:34:52.213 DEBUG:teuthology.orchestra.run.smithi038:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:34:52.271 DEBUG:teuthology.orchestra.run.smithi038:> stat /dev/vg_nvme/lv_2 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout:Access: 2024-04-26 04:34:37.823434259 +0000 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout:Modify: 2024-04-26 04:28:56.542670398 +0000 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout:Change: 2024-04-26 04:28:56.542670398 +0000 2024-04-26T04:34:52.330 INFO:teuthology.orchestra.run.smithi038.stdout: Birth: 2024-04-26 04:28:56.542670398 +0000 2024-04-26T04:34:52.331 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:34:52.396 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records in 2024-04-26T04:34:52.396 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records out 2024-04-26T04:34:52.396 INFO:teuthology.orchestra.run.smithi038.stderr:512 bytes copied, 0.000440716 s, 1.2 MB/s 2024-04-26T04:34:52.397 DEBUG:teuthology.orchestra.run.smithi038:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:34:52.456 DEBUG:teuthology.orchestra.run.smithi038:> stat /dev/vg_nvme/lv_3 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout:Access: 2024-04-26 04:34:37.823434259 +0000 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout:Modify: 2024-04-26 04:28:56.893664221 +0000 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout:Change: 2024-04-26 04:28:56.893664221 +0000 2024-04-26T04:34:52.514 INFO:teuthology.orchestra.run.smithi038.stdout: Birth: 2024-04-26 04:28:56.893664221 +0000 2024-04-26T04:34:52.514 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:34:52.578 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records in 2024-04-26T04:34:52.579 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records out 2024-04-26T04:34:52.579 INFO:teuthology.orchestra.run.smithi038.stderr:512 bytes copied, 0.000453855 s, 1.1 MB/s 2024-04-26T04:34:52.580 DEBUG:teuthology.orchestra.run.smithi038:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:34:52.636 DEBUG:teuthology.orchestra.run.smithi038:> stat /dev/vg_nvme/lv_4 2024-04-26T04:34:52.692 INFO:teuthology.orchestra.run.smithi038.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout:Access: 2024-04-26 04:34:37.823434259 +0000 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout:Modify: 2024-04-26 04:28:57.252657903 +0000 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout:Change: 2024-04-26 04:28:57.252657903 +0000 2024-04-26T04:34:52.693 INFO:teuthology.orchestra.run.smithi038.stdout: Birth: 2024-04-26 04:28:57.252657903 +0000 2024-04-26T04:34:52.693 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:34:52.757 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records in 2024-04-26T04:34:52.757 INFO:teuthology.orchestra.run.smithi038.stderr:1+0 records out 2024-04-26T04:34:52.757 INFO:teuthology.orchestra.run.smithi038.stderr:512 bytes copied, 0.00036884 s, 1.4 MB/s 2024-04-26T04:34:52.758 DEBUG:teuthology.orchestra.run.smithi038:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:34:52.816 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:34:52.816 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/scratch_devs of=/dev/stdout 2024-04-26T04:34:52.833 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-26T04:34:52.833 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_1 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-04-26 04:34:28.076709054 +0000 2024-04-26T04:34:52.889 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-04-26 04:28:56.191904505 +0000 2024-04-26T04:34:52.890 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-04-26 04:28:56.191904505 +0000 2024-04-26T04:34:52.890 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-04-26 04:28:56.191904505 +0000 2024-04-26T04:34:52.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-26T04:34:52.953 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-04-26T04:34:52.954 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-04-26T04:34:52.954 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000388992 s, 1.3 MB/s 2024-04-26T04:34:52.955 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: Deploying daemon mgr.x on smithi055 2024-04-26T04:34:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[26454]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: Deploying daemon mgr.x on smithi055 2024-04-26T04:34:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:52 smithi038 ceph-mon[36743]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:53.012 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_2 2024-04-26T04:34:53.068 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-26T04:34:53.068 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:53.068 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-26T04:34:53.068 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:53.068 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:53.068 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-04-26 04:34:28.076709054 +0000 2024-04-26T04:34:53.069 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-04-26 04:28:56.546897464 +0000 2024-04-26T04:34:53.069 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-04-26 04:28:56.546897464 +0000 2024-04-26T04:34:53.069 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-04-26 04:28:56.546897464 +0000 2024-04-26T04:34:53.069 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-26T04:34:53.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:34:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-26T04:34:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:34:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: Deploying daemon mgr.x on smithi055 2024-04-26T04:34:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:52 smithi055 ceph-mon[30328]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:53.130 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-04-26T04:34:53.130 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-04-26T04:34:53.130 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000307766 s, 1.7 MB/s 2024-04-26T04:34:53.131 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-26T04:34:53.187 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_3 2024-04-26T04:34:53.242 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-26T04:34:53.242 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:53.242 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-26T04:34:53.243 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:53.243 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:53.243 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-04-26 04:34:28.077709034 +0000 2024-04-26T04:34:53.243 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-04-26 04:28:56.908890284 +0000 2024-04-26T04:34:53.243 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-04-26 04:28:56.908890284 +0000 2024-04-26T04:34:53.243 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-04-26 04:28:56.908890284 +0000 2024-04-26T04:34:53.243 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-26T04:34:53.378 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-04-26T04:34:53.378 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-04-26T04:34:53.379 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000285967 s, 1.8 MB/s 2024-04-26T04:34:53.380 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-26T04:34:53.396 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_4 2024-04-26T04:34:53.456 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-26T04:34:53.456 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-26T04:34:53.456 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 5h/5d Inode: 720 Links: 1 2024-04-26T04:34:53.456 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-26T04:34:53.456 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2024-04-26T04:34:53.457 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2024-04-26 04:34:53.329220952 +0000 2024-04-26T04:34:53.457 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2024-04-26 04:28:57.263883243 +0000 2024-04-26T04:34:53.457 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2024-04-26 04:28:57.263883243 +0000 2024-04-26T04:34:53.457 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: 2024-04-26 04:28:57.263883243 +0000 2024-04-26T04:34:53.457 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-26T04:34:53.584 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2024-04-26T04:34:53.585 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2024-04-26T04:34:53.585 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000373022 s, 1.4 MB/s 2024-04-26T04:34:53.586 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-26T04:34:53.601 INFO:tasks.cephadm:Deploying osd.0 on smithi038 with /dev/vg_nvme/lv_4... 2024-04-26T04:34:53.601 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:34:53.831 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:53 smithi055 systemd[1]: Starting Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:34:53.877 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:54.083 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:53 smithi055 podman[32160]: 2024-04-26 04:34:53.830176466 +0000 UTC m=+0.109158490 container create 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-c61472c) 2024-04-26T04:34:54.083 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:53 smithi055 podman[32160]: 2024-04-26 04:34:53.748923378 +0000 UTC m=+0.027905412 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T04:34:54.083 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:53 smithi055 podman[32160]: 2024-04-26 04:34:53.982460429 +0000 UTC m=+0.261442452 container init 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-c61472c) 2024-04-26T04:34:54.083 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:53 smithi055 podman[32160]: 2024-04-26 04:34:53.985112097 +0000 UTC m=+0.264094122 container start 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-26T04:34:54.083 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mgr[32179]: -- 172.21.15.55:0/405280468 <== mon.1 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5644cdfb69c0 con 0x5644cdf8d000 2024-04-26T04:34:54.083 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 bash[32160]: 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 2024-04-26T04:34:54.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 systemd[1]: Started Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:34:54.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:54.132+0000 7f1cbb7271c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:34:54.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:54.300+0000 7f1cbb7271c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:34:54.767 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:54.408+0000 7f1cbb7271c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:34:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[26454]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[36743]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:54.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:54 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:55.067 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:54.766+0000 7f1cbb7271c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:54.851+0000 7f1cbb7271c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mon[30328]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:55.068 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:54 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:55.723 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:34:55.724 INFO:teuthology.orchestra.run.smithi038.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:34:55.724 INFO:teuthology.orchestra.run.smithi038.stderr: stderr: 10+0 records in 2024-04-26T04:34:55.724 INFO:teuthology.orchestra.run.smithi038.stderr:10+0 records out 2024-04-26T04:34:55.724 INFO:teuthology.orchestra.run.smithi038.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0152325 s, 688 MB/s 2024-04-26T04:34:55.724 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping successful for: 2024-04-26T04:34:56.344 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi038:vg_nvme/lv_4 2024-04-26T04:34:56.447 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:56.120+0000 7f1cbb7271c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:34:56.447 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:56.291+0000 7f1cbb7271c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:34:56.656 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.656 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.656 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:56.656 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:56.656 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: Reconfiguring daemon mgr.y on smithi038 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:56.657 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: Reconfiguring daemon mgr.y on smithi038 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:56.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:56.465+0000 7f1cbb7271c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:56.735+0000 7f1cbb7271c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: Reconfiguring mgr.y (unknown last config time)... 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:34:56.737 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:56.738 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: Reconfiguring daemon mgr.y on smithi038 2024-04-26T04:34:56.738 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:56.738 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:57.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:56.828+0000 7f1cbb7271c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:34:57.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:56.924+0000 7f1cbb7271c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:34:57.379 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:34:57.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:57.207+0000 7f1cbb7271c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:34:58.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:57.689+0000 7f1cbb7271c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:34:58.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-mon[30328]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:58.229+0000 7f1cbb7271c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:58.324+0000 7f1cbb7271c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:34:58.508 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:58.506+0000 7f1cbb7271c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:34:58.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:58.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:58.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:58.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[26454]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:34:58.686 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:58 smithi038 ceph-mon[36743]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:34:59.073 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:58.808+0000 7f1cbb7271c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:34:59.073 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:59.072+0000 7f1cbb7271c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:34:59.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:59.316+0000 7f1cbb7271c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:34:59.679 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[26454]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:59.679 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:34:59.679 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:34:59.679 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:59.680 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[36743]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:59.680 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:34:59.680 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:34:59.680 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:34:59 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:59.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-mon[30328]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:34:59.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:34:59.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:34:59.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:34:59.759 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:59.451+0000 7f1cbb7271c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:34:59.760 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:59.547+0000 7f1cbb7271c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:34:59.760 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:59.666+0000 7f1cbb7271c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:34:59.760 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:34:59 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:34:59.758+0000 7f1cbb7271c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:35:00.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:00 smithi038 ceph-mon[26454]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:00.732 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:00 smithi038 ceph-mon[36743]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:00.783 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:00 smithi055 ceph-mon[30328]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:00.783 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:35:00 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:35:00.586+0000 7f1cbb7271c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:35:00.784 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:35:00 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:35:00.782+0000 7f1cbb7271c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:35:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: Standby manager daemon x started 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3543988458' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01bfbe1d-b610-4e51-bebd-99416d020dab"}]: dispatch 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3543988458' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01bfbe1d-b610-4e51-bebd-99416d020dab"}]': finished 2024-04-26T04:35:01.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:35:01.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:01 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:01.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: Standby manager daemon x started 2024-04-26T04:35:01.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:35:01.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:35:01.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3543988458' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01bfbe1d-b610-4e51-bebd-99416d020dab"}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3543988458' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01bfbe1d-b610-4e51-bebd-99416d020dab"}]': finished 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: Standby manager daemon x started 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:35:01.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/147879862' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:35:01.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3543988458' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "01bfbe1d-b610-4e51-bebd-99416d020dab"}]: dispatch 2024-04-26T04:35:01.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3543988458' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "01bfbe1d-b610-4e51-bebd-99416d020dab"}]': finished 2024-04-26T04:35:01.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: osdmap e5: 1 total, 0 up, 1 in 2024-04-26T04:35:01.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:01 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:02 smithi055 ceph-mon[30328]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-26T04:35:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:35:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:02 smithi055 ceph-mon[30328]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:02 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3276456844' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:02.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[36743]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-26T04:35:02.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:35:02.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[36743]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:02.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3276456844' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:02.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[26454]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-26T04:35:02.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:35:02.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[26454]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:02.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:02 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3276456844' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:04.908 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:04 smithi038 ceph-mon[26454]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:04.908 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:04 smithi038 ceph-mon[36743]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:05.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:04 smithi055 ceph-mon[30328]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:06.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:06 smithi038 ceph-mon[26454]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:06.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:06 smithi038 ceph-mon[36743]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:07.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:06 smithi055 ceph-mon[30328]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:07 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:35:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:07 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:07 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:35:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:07 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:08.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:07 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:35:08.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:07 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:09.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:08 smithi055 ceph-mon[30328]: Deploying daemon osd.0 on smithi038 2024-04-26T04:35:09.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:08 smithi055 ceph-mon[30328]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:09.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:08 smithi038 ceph-mon[26454]: Deploying daemon osd.0 on smithi038 2024-04-26T04:35:09.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:08 smithi038 ceph-mon[26454]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:09.189 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:08 smithi038 ceph-mon[36743]: Deploying daemon osd.0 on smithi038 2024-04-26T04:35:09.189 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:08 smithi038 ceph-mon[36743]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:10.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:10 smithi038 ceph-mon[36743]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:10.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:10 smithi038 ceph-mon[26454]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:11.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:10 smithi055 ceph-mon[30328]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:13.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:12 smithi055 ceph-mon[30328]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:13.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:13.161 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[26454]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:13.161 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:13.162 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:13.162 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:13.162 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[36743]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:13.162 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:13.162 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:13.162 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:15.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:14 smithi055 ceph-mon[30328]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:15.133 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:14 smithi038 ceph-mon[26454]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:15.133 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:14 smithi038 ceph-mon[36743]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.303 INFO:teuthology.orchestra.run.smithi038.stdout:Created osd(s) 0 on host 'smithi038' 2024-04-26T04:35:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:35:16.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:16 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:16 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:16.949 DEBUG:teuthology.orchestra.run.smithi038:osd.0> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.0.service 2024-04-26T04:35:16.951 INFO:tasks.cephadm:Deploying osd.1 on smithi038 with /dev/vg_nvme/lv_3... 2024-04-26T04:35:16.951 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:35:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[26454]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:35:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[26454]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:35:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[26454]: from='osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[26454]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[36743]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[36743]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[36743]: from='osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-mon[36743]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:17 smithi055 ceph-mon[30328]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:35:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:17 smithi055 ceph-mon[30328]: osdmap e6: 1 total, 0 up, 1 in 2024-04-26T04:35:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:17 smithi055 ceph-mon[30328]: from='osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:17 smithi055 ceph-mon[30328]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:17.974 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[40673]: 2024-04-26T04:35:17.715+0000 7f07827b2700 -1 osd.0 0 waiting for initial osdmap 2024-04-26T04:35:17.974 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:35:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[40673]: 2024-04-26T04:35:17.718+0000 7f077f5b1700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:35:18.107 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[26454]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[26454]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[26454]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[36743]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[36743]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:18.955 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:18.956 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:18 smithi038 ceph-mon[36743]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:19.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:18 smithi055 ceph-mon[30328]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:35:19.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:18 smithi055 ceph-mon[30328]: osdmap e7: 1 total, 0 up, 1 in 2024-04-26T04:35:19.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:18 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:19.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:18 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:19.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:18 smithi055 ceph-mon[30328]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-26T04:35:19.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:35:19.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[26454]: osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679] boot 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[26454]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:35:19.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:19.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[36743]: osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679] boot 2024-04-26T04:35:19.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[36743]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:35:19.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:19 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:20.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:19 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:35:20.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:19 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:35:20.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:19 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:20.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:19 smithi055 ceph-mon[30328]: osd.0 [v2:172.21.15.38:6802/3619308679,v1:172.21.15.38:6803/3619308679] boot 2024-04-26T04:35:20.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:19 smithi055 ceph-mon[30328]: osdmap e8: 1 total, 1 up, 1 in 2024-04-26T04:35:20.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:19 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:35:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:35:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:21.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:21.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:21.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:21.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: osdmap e9: 1 total, 1 up, 1 in 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:21.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:21.131 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:35:21.131 INFO:teuthology.orchestra.run.smithi038.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:35:21.131 INFO:teuthology.orchestra.run.smithi038.stderr: stderr: 10+0 records in 2024-04-26T04:35:21.131 INFO:teuthology.orchestra.run.smithi038.stderr:10+0 records out 2024-04-26T04:35:21.131 INFO:teuthology.orchestra.run.smithi038.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012395 s, 846 MB/s 2024-04-26T04:35:21.131 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping successful for: 2024-04-26T04:35:21.721 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi038:vg_nvme/lv_3 2024-04-26T04:35:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:21 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi038 2024-04-26T04:35:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:21 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi038 to 16069M 2024-04-26T04:35:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:21 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi038 2024-04-26T04:35:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:21 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi038 to 16069M 2024-04-26T04:35:21.981 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:21 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi038 2024-04-26T04:35:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:21 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi038 to 16069M 2024-04-26T04:35:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:22 smithi055 ceph-mon[30328]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:23.107 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:22 smithi038 ceph-mon[26454]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:23.107 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:22 smithi038 ceph-mon[36743]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:24.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:24.128 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:24.128 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:24.128 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:24.128 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:24.128 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:24.129 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:25.054 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:24 smithi038 ceph-mon[36743]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:25.054 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:24 smithi038 ceph-mon[36743]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:25.055 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:24 smithi038 ceph-mon[26454]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:25.055 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:24 smithi038 ceph-mon[26454]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:24 smithi055 ceph-mon[30328]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:24 smithi055 ceph-mon[30328]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:27.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:26 smithi055 ceph-mon[30328]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:27.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:26 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1221831161' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f8dd4ef-f2b7-41af-8cbd-26faa274219a"}]: dispatch 2024-04-26T04:35:27.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:26 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1221831161' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f8dd4ef-f2b7-41af-8cbd-26faa274219a"}]': finished 2024-04-26T04:35:27.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:26 smithi055 ceph-mon[30328]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:35:27.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:26 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[36743]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1221831161' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f8dd4ef-f2b7-41af-8cbd-26faa274219a"}]: dispatch 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1221831161' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f8dd4ef-f2b7-41af-8cbd-26faa274219a"}]': finished 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[36743]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[26454]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1221831161' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f8dd4ef-f2b7-41af-8cbd-26faa274219a"}]: dispatch 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1221831161' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f8dd4ef-f2b7-41af-8cbd-26faa274219a"}]': finished 2024-04-26T04:35:27.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[26454]: osdmap e10: 2 total, 1 up, 2 in 2024-04-26T04:35:27.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:26 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:28.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:27 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3181441411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:28.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:27 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3181441411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3181441411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:29.050 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:28 smithi038 ceph-mon[26454]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:29.050 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:28 smithi038 ceph-mon[36743]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:29.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:28 smithi055 ceph-mon[30328]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:30.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:30 smithi038 ceph-mon[36743]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:30.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:30 smithi038 ceph-mon[26454]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:31.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:30 smithi055 ceph-mon[30328]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:33.073 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:32 smithi038 ceph-mon[26454]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:33.073 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:35:33.073 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:33.073 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:32 smithi038 ceph-mon[36743]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:33.073 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:35:33.073 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:33.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:32 smithi055 ceph-mon[30328]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:33.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:35:33.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:33 smithi038 ceph-mon[26454]: Deploying daemon osd.1 on smithi038 2024-04-26T04:35:33.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:33 smithi038 ceph-mon[36743]: Deploying daemon osd.1 on smithi038 2024-04-26T04:35:34.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:33 smithi055 ceph-mon[30328]: Deploying daemon osd.1 on smithi038 2024-04-26T04:35:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:34 smithi055 ceph-mon[30328]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:35.222 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:34 smithi038 ceph-mon[26454]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:35.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:34 smithi038 ceph-mon[36743]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:36.892 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:36 smithi038 ceph-mon[26454]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:36.893 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:36 smithi038 ceph-mon[36743]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:37.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:36 smithi055 ceph-mon[30328]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:38 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:38.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:38 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:38.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:38 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:38.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:38 smithi055 ceph-mon[30328]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[36743]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:38.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:38.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:38.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:38.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:38 smithi038 ceph-mon[26454]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:40.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:40.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[26454]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:40.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:35:40.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:40 smithi038 ceph-mon[36743]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:35:41.075 INFO:teuthology.orchestra.run.smithi038.stdout:Created osd(s) 1 on host 'smithi038' 2024-04-26T04:35:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:35:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:40 smithi055 ceph-mon[30328]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:35:41.712 DEBUG:teuthology.orchestra.run.smithi038:osd.1> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.1.service 2024-04-26T04:35:41.714 INFO:tasks.cephadm:Deploying osd.2 on smithi038 with /dev/vg_nvme/lv_2... 2024-04-26T04:35:41.714 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:35:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:35:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:35:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:42 smithi055 ceph-mon[30328]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[26454]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: osdmap e11: 2 total, 1 up, 2 in 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:42.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-mon[36743]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:35:42.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[46663]: 2024-04-26T04:35:42.275+0000 7f2538f0c700 -1 osd.1 0 waiting for initial osdmap 2024-04-26T04:35:42.476 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:35:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[46663]: 2024-04-26T04:35:42.282+0000 7f2532d05700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:35:42.860 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:43.178 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[26454]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[26454]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[26454]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[36743]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[36743]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[36743]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:43.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:43 smithi055 ceph-mon[30328]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:43.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:43 smithi055 ceph-mon[30328]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:35:43.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:43 smithi055 ceph-mon[30328]: osdmap e12: 2 total, 1 up, 2 in 2024-04-26T04:35:43.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:43.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:35:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:35:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[26454]: osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257] boot 2024-04-26T04:35:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[26454]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:35:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:35:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:35:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[36743]: osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257] boot 2024-04-26T04:35:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[36743]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:35:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:44 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:44 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:35:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:44 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:35:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:44 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:44 smithi055 ceph-mon[30328]: osd.1 [v2:172.21.15.38:6810/2181836257,v1:172.21.15.38:6811/2181836257] boot 2024-04-26T04:35:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:44 smithi055 ceph-mon[30328]: osdmap e13: 2 total, 2 up, 2 in 2024-04-26T04:35:44.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:44 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:35:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:45 smithi038 ceph-mon[26454]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:45 smithi038 ceph-mon[26454]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:35:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:45 smithi038 ceph-mon[36743]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:45 smithi038 ceph-mon[36743]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:35:45.269 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:35:45.270 INFO:teuthology.orchestra.run.smithi038.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:35:45.270 INFO:teuthology.orchestra.run.smithi038.stderr: stderr: 10+0 records in 2024-04-26T04:35:45.270 INFO:teuthology.orchestra.run.smithi038.stderr:10+0 records out 2024-04-26T04:35:45.270 INFO:teuthology.orchestra.run.smithi038.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0165209 s, 635 MB/s 2024-04-26T04:35:45.270 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping successful for: 2024-04-26T04:35:45.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:45 smithi055 ceph-mon[30328]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-26T04:35:45.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:45 smithi055 ceph-mon[30328]: osdmap e14: 2 total, 2 up, 2 in 2024-04-26T04:35:45.902 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi038:vg_nvme/lv_2 2024-04-26T04:35:46.513 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:47 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:35:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:47 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:35:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:48 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi038 2024-04-26T04:35:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:48 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi038 to 8034M 2024-04-26T04:35:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:48 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi038 2024-04-26T04:35:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:48 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi038 to 8034M 2024-04-26T04:35:48.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:48 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi038 2024-04-26T04:35:48.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:48 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi038 to 8034M 2024-04-26T04:35:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:49 smithi055 ceph-mon[30328]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:49 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:49 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:49 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[26454]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[36743]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:35:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:35:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:49 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:50.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:50 smithi038 ceph-mon[26454]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:50 smithi038 ceph-mon[36743]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:50.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:50 smithi055 ceph-mon[30328]: from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:35:51.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:51 smithi055 ceph-mon[30328]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:51.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:51 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/691791592' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed88ae78-f042-46aa-af4a-8e62ed88e391"}]: dispatch 2024-04-26T04:35:51.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:51 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/691791592' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed88ae78-f042-46aa-af4a-8e62ed88e391"}]': finished 2024-04-26T04:35:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:51 smithi055 ceph-mon[30328]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:35:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:35:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[26454]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/691791592' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed88ae78-f042-46aa-af4a-8e62ed88e391"}]: dispatch 2024-04-26T04:35:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/691791592' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed88ae78-f042-46aa-af4a-8e62ed88e391"}]': finished 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[26454]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[36743]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/691791592' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed88ae78-f042-46aa-af4a-8e62ed88e391"}]: dispatch 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/691791592' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed88ae78-f042-46aa-af4a-8e62ed88e391"}]': finished 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[36743]: osdmap e15: 3 total, 2 up, 3 in 2024-04-26T04:35:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:35:52.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:52 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1167992147' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:52.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:52 smithi055 ceph-mon[30328]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:52 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1167992147' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:52 smithi038 ceph-mon[26454]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:52 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1167992147' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:35:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:52 smithi038 ceph-mon[36743]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:54 smithi038 ceph-mon[26454]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:54.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:54 smithi038 ceph-mon[36743]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:55.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:54 smithi055 ceph-mon[30328]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:57.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:56 smithi055 ceph-mon[30328]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:57.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:56 smithi038 ceph-mon[36743]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:56 smithi038 ceph-mon[26454]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:57.917 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:57 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:35:57.917 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:57 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:57.917 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:57 smithi038 ceph-mon[26454]: Deploying daemon osd.2 on smithi038 2024-04-26T04:35:57.917 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:57 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:35:57.918 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:57 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:57.918 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:57 smithi038 ceph-mon[36743]: Deploying daemon osd.2 on smithi038 2024-04-26T04:35:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:57 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:35:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:57 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:35:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:57 smithi055 ceph-mon[30328]: Deploying daemon osd.2 on smithi038 2024-04-26T04:35:59.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:35:58 smithi038 ceph-mon[26454]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:59.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:35:58 smithi038 ceph-mon[36743]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:35:59.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:35:58 smithi055 ceph-mon[30328]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:00.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:00 smithi038 ceph-mon[36743]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:00.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:00 smithi038 ceph-mon[26454]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:01.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:00 smithi055 ceph-mon[30328]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[26454]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[36743]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:03.103 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:03.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:02 smithi055 ceph-mon[30328]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:03.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:03.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:03.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:05.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:04 smithi055 ceph-mon[30328]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:05.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:05.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:05.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.149 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[36743]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:05.149 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.149 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.149 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[26454]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:05.150 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:05.226 INFO:teuthology.orchestra.run.smithi038.stdout:Created osd(s) 2 on host 'smithi038' 2024-04-26T04:36:05.759 DEBUG:teuthology.orchestra.run.smithi038:osd.2> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.2.service 2024-04-26T04:36:05.761 INFO:tasks.cephadm:Deploying osd.3 on smithi038 with /dev/vg_nvme/lv_1... 2024-04-26T04:36:05.761 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:36:06.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:05 smithi055 ceph-mon[30328]: from='osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:06.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:05 smithi055 ceph-mon[30328]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:06.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:05 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:06.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:05 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:06.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:05 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[26454]: from='osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[26454]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[36743]: from='osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[36743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:36:06.107 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:06.108 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:06.108 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:05 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:06.955 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:07.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:06 smithi055 ceph-mon[30328]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:07.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:06 smithi055 ceph-mon[30328]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:36:07.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:06 smithi055 ceph-mon[30328]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:36:07.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:06 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:07.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:06 smithi055 ceph-mon[30328]: from='osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:07.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:06 smithi055 ceph-mon[30328]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:07.206 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[26454]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:07.206 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[26454]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:36:07.206 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[26454]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:36:07.206 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:07.206 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[26454]: from='osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:07.206 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[26454]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:07.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[36743]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:07.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[36743]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:36:07.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[36743]: osdmap e16: 3 total, 2 up, 3 in 2024-04-26T04:36:07.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:07.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[36743]: from='osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:07.207 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-mon[36743]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:07.208 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[52734]: 2024-04-26T04:36:06.852+0000 7fac58db2700 -1 osd.2 0 waiting for initial osdmap 2024-04-26T04:36:07.208 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[52734]: 2024-04-26T04:36:06.855+0000 7fac53bad700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[26454]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[26454]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[36743]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[36743]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:08.200 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:07 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:07 smithi055 ceph-mon[30328]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:36:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:07 smithi055 ceph-mon[30328]: osdmap e17: 3 total, 2 up, 3 in 2024-04-26T04:36:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:07 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:07 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.040 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504] boot 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:09.041 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.042 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504] boot 2024-04-26T04:36:09.042 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:36:09.042 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:08 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:36:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:36:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-26T04:36:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: osd.2 [v2:172.21.15.38:6818/124975504,v1:172.21.15.38:6819/124975504] boot 2024-04-26T04:36:09.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: osdmap e18: 3 total, 3 up, 3 in 2024-04-26T04:36:09.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:08 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:36:09.623 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:36:09.623 INFO:teuthology.orchestra.run.smithi038.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:36:09.623 INFO:teuthology.orchestra.run.smithi038.stderr: stderr: 10+0 records in 2024-04-26T04:36:09.623 INFO:teuthology.orchestra.run.smithi038.stderr:10+0 records out 2024-04-26T04:36:09.623 INFO:teuthology.orchestra.run.smithi038.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0157387 s, 666 MB/s 2024-04-26T04:36:09.623 INFO:teuthology.orchestra.run.smithi038.stderr:--> Zapping successful for: 2024-04-26T04:36:10.322 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi038:vg_nvme/lv_1 2024-04-26T04:36:10.581 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:36:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:36:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:10 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:36:10.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: osdmap e19: 3 total, 3 up, 3 in 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.705 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:10.706 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:10.706 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:10.706 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:10.706 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:10 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi038 2024-04-26T04:36:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi038 to 5356M 2024-04-26T04:36:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:36:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:36:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:36:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:11 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:36:11.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi038 2024-04-26T04:36:11.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi038 to 5356M 2024-04-26T04:36:11.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:11.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:36:11.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi038 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi038 to 5356M 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: osdmap e20: 3 total, 3 up, 3 in 2024-04-26T04:36:11.645 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:11 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:36:12.767 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57049]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:12.767 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57049]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:12.768 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57049]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:12.768 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57049]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:13.085 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57108]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:13.085 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57108]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:13.085 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57108]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:13.085 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:36:12 smithi038 sudo[57108]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[26454]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[26454]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:13.086 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[36743]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:13.087 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:36:13.087 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[36743]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:36:13.087 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:13.087 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:13.087 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:12 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:13.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:12 smithi055 ceph-mon[30328]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:36:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:12 smithi055 ceph-mon[30328]: osdmap e21: 3 total, 3 up, 3 in 2024-04-26T04:36:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:13.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:12 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:13.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57132]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:13.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57132]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:13.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57132]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:13.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57132]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:14.068 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:13 smithi038 ceph-mon[26454]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:14.068 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:13 smithi038 ceph-mon[26454]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:36:14.068 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:13 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:14.069 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:13 smithi038 ceph-mon[36743]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:14.069 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:13 smithi038 ceph-mon[36743]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:36:14.069 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:13 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:14.069 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57156]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:14.069 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57156]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:14.069 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57156]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:14.069 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:36:13 smithi038 sudo[57156]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:14.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:13 smithi055 ceph-mon[30328]: from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi038:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:14.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:13 smithi055 ceph-mon[30328]: osdmap e22: 3 total, 3 up, 3 in 2024-04-26T04:36:14.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:13 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:14.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57168]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:14.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57168]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:14.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57168]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:14.726 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57168]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:15.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57214]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:15.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57214]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:15.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57214]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:15.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:36:14 smithi038 sudo[57214]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:15.678 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57218]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:15.678 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57218]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:15.678 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57218]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:15.678 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57218]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:15.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:15 smithi038 ceph-mon[36743]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57221]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57221]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57221]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 sudo[57221]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:15 smithi038 ceph-mon[26454]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:16.583 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32881]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:16.583 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32881]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:16.583 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32881]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:16.583 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32881]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:16.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32884]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:16.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32884]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:16.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32884]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:16.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 sudo[32884]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:16.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:16 smithi055 ceph-mon[30328]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:17.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57258]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-26T04:36:17.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57258]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:17.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57258]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:17.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57258]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1926313949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]': finished 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57308]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-26T04:36:17.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57308]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-26T04:36:17.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57308]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-26T04:36:17.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 sudo[57308]: pam_unix(sudo:session): session closed for user root 2024-04-26T04:36:17.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1926313949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]': finished 2024-04-26T04:36:17.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1926313949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91b59c6c-a9ca-469d-b8a5-e966b283ef3b"}]': finished 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: osdmap e23: 4 total, 3 up, 4 in 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:36:17.977 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:36:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:18 smithi055 ceph-mon[30328]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:18 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/456990488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:18 smithi055 ceph-mon[30328]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:18 smithi055 ceph-mon[30328]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:18.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[26454]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:18.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/456990488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:18.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[26454]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:18.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[26454]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:18.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[36743]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-26T04:36:18.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/456990488' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:18.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[36743]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-26T04:36:18.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:18 smithi038 ceph-mon[36743]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:20.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:20 smithi038 ceph-mon[26454]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T04:36:20.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:20 smithi038 ceph-mon[36743]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T04:36:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:20 smithi055 ceph-mon[30328]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-26T04:36:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:22 smithi055 ceph-mon[30328]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:36:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:22 smithi038 ceph-mon[26454]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:36:23.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:22 smithi038 ceph-mon[36743]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-26T04:36:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:36:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:24.177 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:36:24.177 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:24.177 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:36:24.177 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:24 smithi055 ceph-mon[30328]: Deploying daemon osd.3 on smithi038 2024-04-26T04:36:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:24 smithi055 ceph-mon[30328]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:36:25.176 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:24 smithi038 ceph-mon[26454]: Deploying daemon osd.3 on smithi038 2024-04-26T04:36:25.177 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:24 smithi038 ceph-mon[26454]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:36:25.177 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:24 smithi038 ceph-mon[36743]: Deploying daemon osd.3 on smithi038 2024-04-26T04:36:25.177 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:24 smithi038 ceph-mon[36743]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-26T04:36:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:26 smithi038 ceph-mon[26454]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:36:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:26 smithi038 ceph-mon[36743]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:36:27.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:26 smithi055 ceph-mon[30328]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-26T04:36:29.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[36743]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-04-26T04:36:29.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:29.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:29.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:29.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[26454]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-04-26T04:36:29.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:29.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:29.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:28 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:28 smithi055 ceph-mon[30328]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-04-26T04:36:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:28 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:28 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:28 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:31.128 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:30 smithi038 ceph-mon[26454]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:36:31.128 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:30 smithi038 ceph-mon[36743]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:36:31.210 INFO:teuthology.orchestra.run.smithi038.stdout:Created osd(s) 3 on host 'smithi038' 2024-04-26T04:36:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:30 smithi055 ceph-mon[30328]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-26T04:36:31.867 DEBUG:teuthology.orchestra.run.smithi038:osd.3> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.3.service 2024-04-26T04:36:31.878 INFO:tasks.cephadm:Deploying osd.4 on smithi055 with /dev/vg_nvme/lv_4... 2024-04-26T04:36:31.878 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-26T04:36:32.146 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:36:32.451 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.451 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.451 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:32 smithi055 ceph-mon[30328]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:36:32.451 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:32 smithi055 ceph-mon[30328]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:32.451 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.451 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[26454]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[26454]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[36743]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:36:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[36743]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:33.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:33 smithi055 ceph-mon[30328]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:36:33.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:33 smithi055 ceph-mon[30328]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:36:33.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:33 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:33.374 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:33 smithi055 ceph-mon[30328]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[26454]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:36:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[26454]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:36:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[26454]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[36743]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:36:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[36743]: osdmap e24: 4 total, 3 up, 4 in 2024-04-26T04:36:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-mon[36743]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:36:33.476 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[58918]: 2024-04-26T04:36:33.185+0000 7ff841788700 -1 osd.3 0 waiting for initial osdmap 2024-04-26T04:36:33.476 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:36:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[58918]: 2024-04-26T04:36:33.191+0000 7ff83a57f700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:36:33.919 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-26T04:36:33.920 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-26T04:36:33.920 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-04-26T04:36:33.920 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-04-26T04:36:33.920 INFO:teuthology.orchestra.run.smithi055.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0147942 s, 709 MB/s 2024-04-26T04:36:33.920 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-04-26T04:36:34.454 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:34 smithi055 ceph-mon[30328]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:36:34.455 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:34 smithi055 ceph-mon[30328]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:36:34.455 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:34 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:34.455 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:34 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:34.455 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:34 smithi055 ceph-mon[30328]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:34.456 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_4 2024-04-26T04:36:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[26454]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[26454]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[26454]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[36743]: from='osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]': finished 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[36743]: osdmap e25: 4 total, 3 up, 4 in 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:34 smithi038 ceph-mon[36743]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-26T04:36:34.763 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:36:35.320 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:36:35.320 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:36:35.320 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:35.320 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[36743]: osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327] boot 2024-04-26T04:36:35.320 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[36743]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:36:35.320 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:35.321 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:36:35.321 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:36:35.321 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:35.321 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[26454]: osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327] boot 2024-04-26T04:36:35.321 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[26454]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:36:35.321 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:35 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:35 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:36:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:35 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:36:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:35 smithi055 ceph-mon[30328]: osd.3 [v2:172.21.15.38:6826/1698863327,v1:172.21.15.38:6827/1698863327] boot 2024-04-26T04:36:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:35 smithi055 ceph-mon[30328]: osdmap e26: 4 total, 4 up, 4 in 2024-04-26T04:36:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:35 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:36:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:36:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi038 2024-04-26T04:36:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi038 to 4017M 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:36.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:36 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:36:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi038 2024-04-26T04:36:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi038 to 4017M 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: osdmap e27: 4 total, 4 up, 4 in 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi038 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi038 to 4017M 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:36.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:36.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:36.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:36.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:36.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:36.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:36 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[36743]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:37.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[26454]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:37.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:37.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:37.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:37 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:37.566 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:37 smithi055 ceph-mon[30328]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:36:37.566 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:37 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:36:37.566 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:37 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:36:37.566 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:37 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:38 smithi055 ceph-mon[30328]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:38.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:38 smithi038 ceph-mon[26454]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:38.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:38 smithi038 ceph-mon[36743]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:39 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/4137583650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]: dispatch 2024-04-26T04:36:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:39 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]: dispatch 2024-04-26T04:36:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:39 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]': finished 2024-04-26T04:36:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:39 smithi055 ceph-mon[30328]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:36:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:39 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/4137583650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]: dispatch 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]: dispatch 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]': finished 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[36743]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/4137583650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]: dispatch 2024-04-26T04:36:39.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]: dispatch 2024-04-26T04:36:39.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2c7373f-2b75-4f2e-b028-d5c1dabcee4a"}]': finished 2024-04-26T04:36:39.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[26454]: osdmap e28: 5 total, 4 up, 5 in 2024-04-26T04:36:39.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:39 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:40 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/2551796741' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:40.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:40 smithi055 ceph-mon[30328]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:40 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/2551796741' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:40 smithi038 ceph-mon[26454]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:40 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/2551796741' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:36:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:40 smithi038 ceph-mon[36743]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:43.077 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:42 smithi055 ceph-mon[30328]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:42 smithi038 ceph-mon[26454]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:43.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:42 smithi038 ceph-mon[36743]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:44.946 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:44 smithi055 ceph-mon[30328]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:44 smithi038 ceph-mon[26454]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:45.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:44 smithi038 ceph-mon[36743]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:46.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:36:46.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:46.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:45 smithi055 ceph-mon[30328]: Deploying daemon osd.4 on smithi055 2024-04-26T04:36:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:36:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:45 smithi038 ceph-mon[26454]: Deploying daemon osd.4 on smithi055 2024-04-26T04:36:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:36:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:45 smithi038 ceph-mon[36743]: Deploying daemon osd.4 on smithi055 2024-04-26T04:36:47.027 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:46 smithi055 ceph-mon[30328]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:47.125 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:46 smithi038 ceph-mon[26454]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:47.126 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:46 smithi038 ceph-mon[36743]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:49.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:48 smithi038 ceph-mon[36743]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:48 smithi038 ceph-mon[26454]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:49.332 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:48 smithi055 ceph-mon[30328]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:51.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:50 smithi055 ceph-mon[30328]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:51.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:51.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:51.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:50 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[36743]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:51.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[26454]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:51.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:51.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:51.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:50 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:52 smithi038 ceph-mon[26454]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:52 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:53.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:52 smithi038 ceph-mon[36743]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:53.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:53.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:52 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:53.317 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:52 smithi055 ceph-mon[30328]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:53.318 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:53.318 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:52 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:53.631 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 4 on host 'smithi055' 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:54.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:53 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:54.239 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:53 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:36:54.264 DEBUG:teuthology.orchestra.run.smithi055:osd.4> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.4.service 2024-04-26T04:36:54.267 INFO:tasks.cephadm:Deploying osd.5 on smithi055 with /dev/vg_nvme/lv_3... 2024-04-26T04:36:54.267 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-26T04:36:55.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:54 smithi055 ceph-mon[30328]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:55.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:54 smithi055 ceph-mon[30328]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:36:55.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:54 smithi055 ceph-mon[30328]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:36:55.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:54 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:55.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:54 smithi055 ceph-mon[30328]: from='osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:36:55.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:54 smithi055 ceph-mon[30328]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:36:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[26454]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[26454]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:36:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[26454]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:36:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[26454]: from='osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:36:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[26454]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:36:55.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[36743]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:55.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[36743]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-26T04:36:55.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[36743]: osdmap e29: 5 total, 4 up, 5 in 2024-04-26T04:36:55.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:55.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[36743]: from='osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:36:55.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:54 smithi038 ceph-mon[36743]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:36:55.421 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 04:36:55 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T04:36:55.155+0000 7fc5467ee700 -1 osd.4 0 waiting for initial osdmap 2024-04-26T04:36:55.421 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 04:36:55 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T04:36:55.159+0000 7fc53f5e5700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:36:55.695 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414] boot 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.346 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:56 smithi055 ceph-mon[30328]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414] boot 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[36743]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: osdmap e30: 5 total, 4 up, 5 in 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: osd.4 [v2:172.21.15.55:6800/3196578414,v1:172.21.15.55:6801/3196578414] boot 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: osdmap e31: 5 total, 5 up, 5 in 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:36:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:56 smithi038 ceph-mon[26454]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-26T04:36:57.235 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:57 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:36:57.235 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:57 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:36:57.235 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:57 smithi055 ceph-mon[30328]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:36:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:57 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:36:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:57 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:36:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:57 smithi038 ceph-mon[26454]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:36:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:57 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:36:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:57 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:36:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:57 smithi038 ceph-mon[36743]: osdmap e32: 5 total, 5 up, 5 in 2024-04-26T04:36:57.700 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-26T04:36:57.700 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-26T04:36:57.700 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-04-26T04:36:57.700 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-04-26T04:36:57.700 INFO:teuthology.orchestra.run.smithi055.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.010411 s, 1.0 GB/s 2024-04-26T04:36:57.700 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-04-26T04:36:58.298 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_3 2024-04-26T04:36:58.557 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:36:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi055 2024-04-26T04:36:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi055 to 17093M 2024-04-26T04:36:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[36743]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi055 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi055 to 17093M 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:58.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:58.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:36:58.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:58 smithi038 ceph-mon[26454]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi055 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi055 to 17093M 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:36:58.979 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:36:58.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:36:58.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: osdmap e33: 5 total, 5 up, 5 in 2024-04-26T04:36:58.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:58 smithi055 ceph-mon[30328]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:36:59.816 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:36:59 smithi055 ceph-mon[30328]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:36:59.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:36:59 smithi038 ceph-mon[26454]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:36:59.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:36:59 smithi038 ceph-mon[36743]: osdmap e34: 5 total, 5 up, 5 in 2024-04-26T04:37:00.956 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:00 smithi055 ceph-mon[30328]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:37:00.956 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:00 smithi055 ceph-mon[30328]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:37:00.957 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:00 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:00.957 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:00 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:00.957 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:00 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[26454]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[26454]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[36743]: osdmap e35: 5 total, 5 up, 5 in 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[36743]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-26T04:37:01.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:01.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:01.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:00 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:01.905 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:01 smithi055 ceph-mon[30328]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:01 smithi038 ceph-mon[26454]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:02.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:01 smithi038 ceph-mon[36743]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:03.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:02 smithi055 ceph-mon[30328]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-04-26T04:37:03.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:02 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/1178738574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]: dispatch 2024-04-26T04:37:03.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:02 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]: dispatch 2024-04-26T04:37:03.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:02 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]': finished 2024-04-26T04:37:03.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:02 smithi055 ceph-mon[30328]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:37:03.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[26454]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-04-26T04:37:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/1178738574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]: dispatch 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]: dispatch 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]': finished 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[26454]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[36743]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 114 KiB/s, 0 objects/s recovering 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/1178738574' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]: dispatch 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]: dispatch 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fddae5a2-7bb9-417d-a4a0-f8bd5aef226a"}]': finished 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[36743]: osdmap e36: 6 total, 5 up, 6 in 2024-04-26T04:37:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:04.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:03 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/2616420081' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:03 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/2616420081' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:03 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/2616420081' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:05.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:04 smithi055 ceph-mon[30328]: pgmap v120: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:05.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:04 smithi038 ceph-mon[26454]: pgmap v120: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:05.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:04 smithi038 ceph-mon[36743]: pgmap v120: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:06.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:06 smithi055 ceph-mon[30328]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:37:07.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:06 smithi038 ceph-mon[26454]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:37:07.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:06 smithi038 ceph-mon[36743]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:37:09.070 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:08 smithi055 ceph-mon[30328]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:09.070 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:08 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:37:09.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:08 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:09.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:08 smithi038 ceph-mon[26454]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:09.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:37:09.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:09.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:08 smithi038 ceph-mon[36743]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:09.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:08 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:37:09.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:08 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:10.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:09 smithi055 ceph-mon[30328]: Deploying daemon osd.5 on smithi055 2024-04-26T04:37:10.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:09 smithi038 ceph-mon[26454]: Deploying daemon osd.5 on smithi055 2024-04-26T04:37:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:09 smithi038 ceph-mon[36743]: Deploying daemon osd.5 on smithi055 2024-04-26T04:37:11.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:10 smithi055 ceph-mon[30328]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:11.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:10 smithi038 ceph-mon[26454]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:11.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:10 smithi038 ceph-mon[36743]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:13.166 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:12 smithi055 ceph-mon[30328]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:12 smithi038 ceph-mon[26454]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:13.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:12 smithi038 ceph-mon[36743]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:15.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:14 smithi055 ceph-mon[30328]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:15.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:14 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:15.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:14 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:15.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:14 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[26454]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:15.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[36743]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:15.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:15.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:15.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:14 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:17.097 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:17 smithi055 ceph-mon[30328]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:17.097 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.098 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.098 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:17.098 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:17.098 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:17 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.276 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[26454]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[36743]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:17.277 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:17.278 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:17 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:17.425 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 5 on host 'smithi055' 2024-04-26T04:37:18.052 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:18 smithi055 ceph-mon[30328]: from='osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:37:18.053 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:18 smithi055 ceph-mon[30328]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:37:18.053 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:18 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:18.053 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:18 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:18.053 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:18 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:18.079 DEBUG:teuthology.orchestra.run.smithi055:osd.5> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.5.service 2024-04-26T04:37:18.081 INFO:tasks.cephadm:Deploying osd.6 on smithi055 with /dev/vg_nvme/lv_2... 2024-04-26T04:37:18.081 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-26T04:37:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[26454]: from='osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:37:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[26454]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:37:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:18.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[36743]: from='osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:37:18.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[36743]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-26T04:37:18.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:18.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:18.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:18 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:18.758 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:19.148 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-mon[30328]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:19.148 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-mon[30328]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:37:19.148 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-mon[30328]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:37:19.149 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:19.149 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-mon[30328]: from='osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:19.149 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-mon[30328]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:19.149 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T04:37:19.039+0000 7fd50584e700 -1 osd.5 0 waiting for initial osdmap 2024-04-26T04:37:19.149 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 04:37:19 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T04:37:19.044+0000 7fd4fe645700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:37:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[26454]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[26454]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:37:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[26454]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:37:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[26454]: from='osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[26454]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[36743]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[36743]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[36743]: osdmap e37: 6 total, 5 up, 6 in 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[36743]: from='osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:19.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:19 smithi038 ceph-mon[36743]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:37:20.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:37:20.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512] boot 2024-04-26T04:37:20.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:37:20.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512] boot 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:37:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: osdmap e38: 6 total, 5 up, 6 in 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: osd.5 [v2:172.21.15.55:6808/801519512,v1:172.21.15.55:6809/801519512] boot 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: osdmap e39: 6 total, 6 up, 6 in 2024-04-26T04:37:20.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:37:21.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:21 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:37:21.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:21 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:37:21.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:21 smithi055 ceph-mon[30328]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:21.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:21 smithi055 ceph-mon[30328]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:37:21.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:21 smithi055 ceph-mon[30328]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:37:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:37:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:37:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[26454]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[26454]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:37:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[26454]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:37:21.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:37:21.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:37:21.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[36743]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-26T04:37:21.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[36743]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T04:37:21.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:21 smithi038 ceph-mon[36743]: osdmap e40: 6 total, 6 up, 6 in 2024-04-26T04:37:21.517 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-26T04:37:21.517 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-26T04:37:21.517 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-04-26T04:37:21.517 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-04-26T04:37:21.517 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.01689 s, 621 MB/s 2024-04-26T04:37:21.518 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-04-26T04:37:22.175 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_2 2024-04-26T04:37:22.771 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:22.894 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:22 smithi055 ceph-mon[30328]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:37:22.894 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:22 smithi055 ceph-mon[30328]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:22.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:22 smithi038 ceph-mon[26454]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:37:22.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:22 smithi038 ceph-mon[26454]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:22.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:22 smithi038 ceph-mon[36743]: osdmap e41: 6 total, 6 up, 6 in 2024-04-26T04:37:22.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:22 smithi038 ceph-mon[36743]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi055 2024-04-26T04:37:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:37:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi055 to 8546M 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi055 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi055 to 8546M 2024-04-26T04:37:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:23.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:23.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:23.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:23 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:24.013 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi055 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: osdmap e42: 6 total, 6 up, 6 in 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi055 to 8546M 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:23 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:24.997 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:24 smithi055 ceph-mon[30328]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:37:24.997 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:24 smithi055 ceph-mon[30328]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:24.997 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:24 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:24.997 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:24 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:24.997 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:24 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[26454]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[26454]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[36743]: osdmap e43: 6 total, 6 up, 6 in 2024-04-26T04:37:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[36743]: pgmap v137: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:25.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:25.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:25.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:24 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:25.921 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:25 smithi055 ceph-mon[30328]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:25 smithi038 ceph-mon[26454]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:25 smithi038 ceph-mon[36743]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:27.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:26 smithi055 ceph-mon[30328]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-26T04:37:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:26 smithi038 ceph-mon[26454]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-26T04:37:27.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:26 smithi038 ceph-mon[36743]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: Cluster is now healthy 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/3456382234' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]: dispatch 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]: dispatch 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]': finished 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:28.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:27 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/3156335771' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: Cluster is now healthy 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/3456382234' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]: dispatch 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]: dispatch 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]': finished 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/3156335771' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: Cluster is now healthy 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/3456382234' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]: dispatch 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]: dispatch 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7d114339-5312-4b95-bb3e-b3a68a38d683"}]': finished 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: osdmap e44: 7 total, 6 up, 7 in 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:27 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/3156335771' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:29.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:28 smithi055 ceph-mon[30328]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:29.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:28 smithi038 ceph-mon[26454]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:29.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:28 smithi038 ceph-mon[36743]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:37:31.036 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:30 smithi055 ceph-mon[30328]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:37:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:30 smithi038 ceph-mon[26454]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:37:31.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:30 smithi038 ceph-mon[36743]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-26T04:37:33.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:32 smithi055 ceph-mon[30328]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:37:33.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:32 smithi038 ceph-mon[26454]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:37:33.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:32 smithi038 ceph-mon[36743]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-26T04:37:34.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:33 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:37:34.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:33 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:34.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:33 smithi055 ceph-mon[30328]: Deploying daemon osd.6 on smithi055 2024-04-26T04:37:34.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:33 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:37:34.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:33 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:34.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:33 smithi038 ceph-mon[26454]: Deploying daemon osd.6 on smithi055 2024-04-26T04:37:34.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:33 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:37:34.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:33 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:34.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:33 smithi038 ceph-mon[36743]: Deploying daemon osd.6 on smithi055 2024-04-26T04:37:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:34 smithi055 ceph-mon[30328]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:37:35.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:34 smithi038 ceph-mon[36743]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:37:35.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:34 smithi038 ceph-mon[26454]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:37:37.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:36 smithi055 ceph-mon[30328]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:37.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:36 smithi038 ceph-mon[26454]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:37.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:36 smithi038 ceph-mon[36743]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:39.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:38 smithi055 ceph-mon[30328]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:39.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:38 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:39.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:38 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:39.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:38 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[26454]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:39.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[36743]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:39.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:38 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:40 smithi055 ceph-mon[30328]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:40 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.201 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 6 on host 'smithi055' 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[36743]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[26454]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:40 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:41.881 DEBUG:teuthology.orchestra.run.smithi055:osd.6> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.6.service 2024-04-26T04:37:41.883 INFO:tasks.cephadm:Deploying osd.7 on smithi055 with /dev/vg_nvme/lv_1... 2024-04-26T04:37:41.883 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-26T04:37:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:41 smithi055 ceph-mon[30328]: from='osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:37:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:41 smithi055 ceph-mon[30328]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:37:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:41 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:41 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:41 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[26454]: from='osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[26454]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[36743]: from='osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:37:42.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[36743]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-26T04:37:42.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:42.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:37:42.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:41 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:42.929 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:43.147 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-mon[30328]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:43.147 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-mon[30328]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:37:43.147 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-mon[30328]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:37:43.148 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:43.148 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-mon[30328]: from='osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:43.148 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-mon[30328]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:43.148 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T04:37:42.880+0000 7f7635758700 -1 osd.6 0 waiting for initial osdmap 2024-04-26T04:37:43.148 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 04:37:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T04:37:42.892+0000 7f7630553700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:37:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[26454]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[26454]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:37:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[26454]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:37:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[26454]: from='osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[26454]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:43.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[36743]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:43.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[36743]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-26T04:37:43.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[36743]: osdmap e45: 7 total, 6 up, 7 in 2024-04-26T04:37:43.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:43.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[36743]: from='osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:43.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:42 smithi038 ceph-mon[36743]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:37:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:43 smithi055 ceph-mon[30328]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:37:44.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:43 smithi055 ceph-mon[30328]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:37:44.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:44.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:43 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[26454]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:37:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[26454]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:37:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:44.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[36743]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:37:44.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[36743]: osdmap e46: 7 total, 6 up, 7 in 2024-04-26T04:37:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:44.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:43 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[26454]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[26454]: osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219] boot 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[26454]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:45.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:37:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:37:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[36743]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[36743]: osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219] boot 2024-04-26T04:37:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[36743]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:37:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:44 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:45.254 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:44 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:37:45.254 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:44 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:37:45.254 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:44 smithi055 ceph-mon[30328]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-26T04:37:45.255 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:44 smithi055 ceph-mon[30328]: osd.6 [v2:172.21.15.55:6816/1074490219,v1:172.21.15.55:6817/1074490219] boot 2024-04-26T04:37:45.255 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:44 smithi055 ceph-mon[30328]: osdmap e47: 7 total, 7 up, 7 in 2024-04-26T04:37:45.255 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:44 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:37:45.717 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-26T04:37:45.717 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-26T04:37:45.717 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2024-04-26T04:37:45.717 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2024-04-26T04:37:45.717 INFO:teuthology.orchestra.run.smithi055.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0162247 s, 646 MB/s 2024-04-26T04:37:45.717 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.025 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:46.025 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:46.025 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:45 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:37:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: osdmap e48: 7 total, 7 up, 7 in 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:37:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:46.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:37:46.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:45 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:37:46.297 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi055:vg_nvme/lv_1 2024-04-26T04:37:46.558 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:37:46.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:46 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi055 2024-04-26T04:37:46.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:46 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi055 to 5697M 2024-04-26T04:37:46.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:46 smithi055 ceph-mon[30328]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:46.980 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:46 smithi055 ceph-mon[30328]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi055 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi055 to 5697M 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[26454]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[26454]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi055 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi055 to 5697M 2024-04-26T04:37:47.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[36743]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:46 smithi038 ceph-mon[36743]: osdmap e49: 7 total, 7 up, 7 in 2024-04-26T04:37:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:48 smithi038 ceph-mon[26454]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:37:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:48 smithi038 ceph-mon[26454]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:48 smithi038 ceph-mon[36743]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:37:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:48 smithi038 ceph-mon[36743]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:48 smithi055 ceph-mon[30328]: osdmap e50: 7 total, 7 up, 7 in 2024-04-26T04:37:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:48 smithi055 ceph-mon[30328]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:49 smithi055 ceph-mon[30328]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:37:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:49 smithi055 ceph-mon[30328]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:49 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:49 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:49 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[26454]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:37:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[26454]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[36743]: osdmap e51: 7 total, 7 up, 7 in 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[36743]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-26T04:37:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:49 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:50 smithi038 ceph-mon[26454]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:50.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:50 smithi038 ceph-mon[36743]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:50.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:50 smithi055 ceph-mon[30328]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:37:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/3668802608' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]: dispatch 2024-04-26T04:37:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]: dispatch 2024-04-26T04:37:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[26454]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]': finished 2024-04-26T04:37:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[26454]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:37:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:37:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/3668802608' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]: dispatch 2024-04-26T04:37:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]: dispatch 2024-04-26T04:37:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[36743]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]': finished 2024-04-26T04:37:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[36743]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:37:51.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:51 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:37:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:51 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/3668802608' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]: dispatch 2024-04-26T04:37:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:51 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]: dispatch 2024-04-26T04:37:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:51 smithi055 ceph-mon[30328]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55f50e10-51c3-4688-bd17-c0abdb973358"}]': finished 2024-04-26T04:37:51.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:51 smithi055 ceph-mon[30328]: osdmap e52: 8 total, 7 up, 8 in 2024-04-26T04:37:51.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:51 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:37:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:52 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/4044755625' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:52 smithi055 ceph-mon[30328]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-26T04:37:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:52 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/4044755625' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:52 smithi038 ceph-mon[26454]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-26T04:37:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:52 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/4044755625' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-26T04:37:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:52 smithi038 ceph-mon[36743]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-04-26T04:37:54.923 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:54 smithi055 ceph-mon[30328]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-26T04:37:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:54 smithi038 ceph-mon[26454]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-26T04:37:55.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:54 smithi038 ceph-mon[36743]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-26T04:37:56.892 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:56 smithi055 ceph-mon[30328]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:37:57.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:56 smithi038 ceph-mon[36743]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:37:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:56 smithi038 ceph-mon[26454]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-26T04:37:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:57 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:37:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:57 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:57 smithi055 ceph-mon[30328]: Deploying daemon osd.7 on smithi055 2024-04-26T04:37:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:57 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:37:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:57 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:57 smithi038 ceph-mon[26454]: Deploying daemon osd.7 on smithi055 2024-04-26T04:37:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:57 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:37:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:57 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:37:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:57 smithi038 ceph-mon[36743]: Deploying daemon osd.7 on smithi055 2024-04-26T04:37:59.085 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:37:58 smithi055 ceph-mon[30328]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T04:37:59.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:37:58 smithi038 ceph-mon[26454]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T04:37:59.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:37:58 smithi038 ceph-mon[36743]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2024-04-26T04:38:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:00 smithi038 ceph-mon[26454]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:01.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:00 smithi038 ceph-mon[36743]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:01.319 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:00 smithi055 ceph-mon[30328]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-26T04:38:02.995 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:02.995 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:02.995 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:02 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:02.995 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:02 smithi055 ceph-mon[30328]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[26454]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:03.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:03.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:02 smithi038 ceph-mon[36743]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:04.593 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 7 on host 'smithi055' 2024-04-26T04:38:05.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:04 smithi055 ceph-mon[30328]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:05.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:04 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.222 DEBUG:teuthology.orchestra.run.smithi055:osd.7> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.7.service 2024-04-26T04:38:05.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:04 smithi038 ceph-mon[26454]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:05.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:04 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:04 smithi038 ceph-mon[36743]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:05.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:04 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.225 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-26T04:38:05.225 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T04:38:05.508 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:05.924 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[26454]: from='osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:05.924 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[26454]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:05.925 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.925 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.925 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[36743]: from='osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:05.925 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[36743]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:05.925 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:05.925 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:05 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:06.264 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:05 smithi055 ceph-mon[30328]: from='osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:06.264 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:05 smithi055 ceph-mon[30328]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-26T04:38:06.265 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:05 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:06.265 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:05 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:06.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[26454]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:06.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[26454]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:38:06.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[26454]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:38:06.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:06.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[26454]: from='osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:38:06.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[26454]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:38:06.945 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[36743]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:06.945 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[36743]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:38:06.945 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[36743]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:38:06.945 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:06.945 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[36743]: from='osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:38:06.945 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:06 smithi038 ceph-mon[36743]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:38:07.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-mon[30328]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:07.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-mon[30328]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-26T04:38:07.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-mon[30328]: osdmap e53: 8 total, 7 up, 8 in 2024-04-26T04:38:07.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:07.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-mon[30328]: from='osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:38:07.171 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-mon[30328]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2024-04-26T04:38:07.171 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T04:38:06.883+0000 7f09f4c4d700 -1 osd.7 0 waiting for initial osdmap 2024-04-26T04:38:07.171 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 04:38:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T04:38:06.891+0000 7f09ef247700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:38:07.362 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[36743]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[36743]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3553485849' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[26454]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:38:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[26454]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:38:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:07 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3553485849' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:08.011 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1714106263,"num_in_osds":8,"osd_in_since":1714106270,"num_remapped_pgs":0} 2024-04-26T04:38:08.270 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:07 smithi055 ceph-mon[30328]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2024-04-26T04:38:08.271 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:07 smithi055 ceph-mon[30328]: osdmap e54: 8 total, 7 up, 8 in 2024-04-26T04:38:08.271 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:07 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:08.271 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:07 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:08.271 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:07 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3553485849' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:09.012 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[36743]: purged_snaps scrub starts 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[36743]: purged_snaps scrub ok 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[36743]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[36743]: osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653] boot 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[36743]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[26454]: purged_snaps scrub starts 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[26454]: purged_snaps scrub ok 2024-04-26T04:38:09.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[26454]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:09.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[26454]: osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653] boot 2024-04-26T04:38:09.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[26454]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:38:09.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:08 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:09.275 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:08 smithi055 ceph-mon[30328]: purged_snaps scrub starts 2024-04-26T04:38:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:08 smithi055 ceph-mon[30328]: purged_snaps scrub ok 2024-04-26T04:38:09.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:08 smithi055 ceph-mon[30328]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-26T04:38:09.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:08 smithi055 ceph-mon[30328]: osd.7 [v2:172.21.15.55:6824/1950122653,v1:172.21.15.55:6825/1950122653] boot 2024-04-26T04:38:09.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:08 smithi055 ceph-mon[30328]: osdmap e55: 8 total, 8 up, 8 in 2024-04-26T04:38:09.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:08 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:38:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: Detected new or changed devices on smithi055 2024-04-26T04:38:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi055 to 4273M 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[36743]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: Detected new or changed devices on smithi055 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi055 to 4273M 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:10.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:10.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:10.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:10.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:09 smithi038 ceph-mon[26454]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:38:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: osdmap e56: 8 total, 8 up, 8 in 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: Detected new or changed devices on smithi055 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi055 to 4273M 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:10.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:10.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:10.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:10.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:10.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:10.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:09 smithi055 ceph-mon[30328]: osdmap e57: 8 total, 8 up, 8 in 2024-04-26T04:38:10.992 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:38:11.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:10 smithi038 ceph-mon[36743]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:11.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:10 smithi038 ceph-mon[36743]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:38:11.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:10 smithi038 ceph-mon[26454]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:11.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:10 smithi038 ceph-mon[26454]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:38:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:10 smithi055 ceph-mon[30328]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:10 smithi055 ceph-mon[30328]: osdmap e58: 8 total, 8 up, 8 in 2024-04-26T04:38:11.502 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1714106287,"num_in_osds":8,"osd_in_since":1714106270,"num_remapped_pgs":0} 2024-04-26T04:38:11.503 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:38:11.768 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:11 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1277659307' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:11 smithi038 ceph-mon[26454]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:38:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:11 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1277659307' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:11 smithi038 ceph-mon[36743]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:38:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:11 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1277659307' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-26T04:38:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:11 smithi055 ceph-mon[30328]: osdmap e59: 8 total, 8 up, 8 in 2024-04-26T04:38:13.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:12 smithi038 ceph-mon[36743]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:12 smithi038 ceph-mon[26454]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:13.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:12 smithi055 ceph-mon[30328]: pgmap v177: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:13.699 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:38:13.699 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":59,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:38:11.707128+0000","last_up_change":"2024-04-26T04:38:07.872498+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6803","nonce":3619308679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6805","nonce":3619308679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6809","nonce":3619308679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6807","nonce":3619308679}]},"public_addr":"172.21.15.38:6803/3619308679","cluster_addr":"172.21.15.38:6805/3619308679","heartbeat_back_addr":"172.21.15.38:6809/3619308679","heartbeat_front_addr":"172.21.15.38:6807/3619308679","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6811","nonce":2181836257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6813","nonce":2181836257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6817","nonce":2181836257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6815","nonce":2181836257}]},"public_addr":"172.21.15.38:6811/2181836257","cluster_addr":"172.21.15.38:6813/2181836257","heartbeat_back_addr":"172.21.15.38:6817/2181836257","heartbeat_front_addr":"172.21.15.38:6815/2181836257","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6819","nonce":124975504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6821","nonce":124975504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6825","nonce":124975504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6823","nonce":124975504}]},"public_addr":"172.21.15.38:6819/124975504","cluster_addr":"172.21.15.38:6821/124975504","heartbeat_back_addr":"172.21.15.38:6825/124975504","heartbeat_front_addr":"172.21.15.38:6823/124975504","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6827","nonce":1698863327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6829","nonce":1698863327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6833","nonce":1698863327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6831","nonce":1698863327}]},"public_addr":"172.21.15.38:6827/1698863327","cluster_addr":"172.21.15.38:6829/1698863327","heartbeat_back_addr":"172.21.15.38:6833/1698863327","heartbeat_front_addr":"172.21.15.38:6831/1698863327","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:38:13.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:13 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/719361608' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:38:13.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:13 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/719361608' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:38:14.259 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-26T04:36:10.250717+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-26T04:38:14.260 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-26T04:38:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:13 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/719361608' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:38:14.519 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:15.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:14 smithi038 ceph-mon[26454]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-26T04:38:15.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:14 smithi038 ceph-mon[36743]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-26T04:38:15.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:14 smithi055 ceph-mon[30328]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-04-26T04:38:16.316 INFO:teuthology.orchestra.run.smithi038.stdout:pg_num: 1 2024-04-26T04:38:17.053 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:16 smithi038 ceph-mon[26454]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:38:17.053 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:16 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/853514627' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:38:17.053 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:16 smithi038 ceph-mon[36743]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:38:17.054 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:16 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/853514627' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:38:17.055 INFO:tasks.cephadm:Adding prometheus.a on smithi055 2024-04-26T04:38:17.055 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi055=a' 2024-04-26T04:38:17.328 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:16 smithi055 ceph-mon[30328]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-26T04:38:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:16 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/853514627' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-26T04:38:19.172 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled prometheus update... 2024-04-26T04:38:19.185 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:18 smithi055 ceph-mon[30328]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-26T04:38:19.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:18 smithi038 ceph-mon[26454]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-26T04:38:19.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:18 smithi038 ceph-mon[36743]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-04-26T04:38:19.853 DEBUG:teuthology.orchestra.run.smithi055:prometheus.a> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-26T04:38:19.855 INFO:tasks.cephadm:Adding node-exporter.a on smithi038 2024-04-26T04:38:19.855 INFO:tasks.cephadm:Adding node-exporter.b on smithi055 2024-04-26T04:38:19.856 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi038=a;smithi055=b' 2024-04-26T04:38:20.155 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:20.460 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:20.460 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: Saving service prometheus spec with placement smithi055=a;count:1 2024-04-26T04:38:20.460 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:20.461 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:20.461 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:20.461 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:20.461 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:20.461 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: Deploying daemon prometheus.a on smithi055 2024-04-26T04:38:20.461 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:20 smithi055 ceph-mon[30328]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: Saving service prometheus spec with placement smithi055=a;count:1 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: Deploying daemon prometheus.a on smithi055 2024-04-26T04:38:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[26454]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: Saving service prometheus spec with placement smithi055=a;count:1 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: Deploying daemon prometheus.a on smithi055 2024-04-26T04:38:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:20 smithi038 ceph-mon[36743]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-26T04:38:22.455 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled node-exporter update... 2024-04-26T04:38:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:22 smithi055 ceph-mon[30328]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-26T04:38:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:22 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:22 smithi038 ceph-mon[26454]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-26T04:38:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:22 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:23.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:22 smithi038 ceph-mon[36743]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-26T04:38:23.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:22 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:23.375 DEBUG:teuthology.orchestra.run.smithi038:node-exporter.a> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-26T04:38:23.378 DEBUG:teuthology.orchestra.run.smithi055:node-exporter.b> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-26T04:38:23.380 INFO:tasks.cephadm:Adding alertmanager.a on smithi038 2024-04-26T04:38:23.381 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi038=a' 2024-04-26T04:38:23.700 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:23 smithi055 ceph-mon[30328]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi038=a;smithi055=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:23 smithi055 ceph-mon[30328]: Saving service node-exporter spec with placement smithi038=a;smithi055=b;count:2 2024-04-26T04:38:24.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:23 smithi038 ceph-mon[36743]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi038=a;smithi055=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:24.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:23 smithi038 ceph-mon[36743]: Saving service node-exporter spec with placement smithi038=a;smithi055=b;count:2 2024-04-26T04:38:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:23 smithi038 ceph-mon[26454]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi038=a;smithi055=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:23 smithi038 ceph-mon[26454]: Saving service node-exporter spec with placement smithi038=a;smithi055=b;count:2 2024-04-26T04:38:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:24 smithi038 ceph-mon[26454]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:38:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:24 smithi038 ceph-mon[36743]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:38:25.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:24 smithi055 ceph-mon[30328]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-26T04:38:26.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:26 smithi055 ceph-mon[30328]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:26 smithi038 ceph-mon[26454]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:27.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:26 smithi038 ceph-mon[36743]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:27.504 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled alertmanager update... 2024-04-26T04:38:28.698 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:28 smithi055 ceph-mon[30328]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi038=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:28.698 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:28 smithi055 ceph-mon[30328]: Saving service alertmanager spec with placement smithi038=a;count:1 2024-04-26T04:38:28.698 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:28 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:28.698 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:28 smithi055 ceph-mon[30328]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[26454]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi038=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[26454]: Saving service alertmanager spec with placement smithi038=a;count:1 2024-04-26T04:38:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[26454]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[36743]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi038=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:38:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[36743]: Saving service alertmanager spec with placement smithi038=a;count:1 2024-04-26T04:38:28.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:28.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:28 smithi038 ceph-mon[36743]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:30.568 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 systemd[1]: Starting Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:38:30.597 DEBUG:teuthology.orchestra.run.smithi038:alertmanager.a> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-26T04:38:30.599 INFO:tasks.cephadm:Adding grafana.a on smithi055 2024-04-26T04:38:30.600 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi055=a' 2024-04-26T04:38:30.851 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 podman[58193]: 2024-04-26 04:38:30.67940115 +0000 UTC m=+0.022467790 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:38:30.851 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 podman[58193]: 2024-04-26 04:38:30.793671334 +0000 UTC m=+0.136737953 container create 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:30.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:30 smithi038 ceph-mon[36743]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:30.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:30 smithi038 ceph-mon[26454]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:31.060 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:31.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-mon[30328]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:31.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 podman[58193]: 2024-04-26 04:38:30.962529167 +0000 UTC m=+0.305595795 container init 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:31.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 podman[58193]: 2024-04-26 04:38:30.965087308 +0000 UTC m=+0.308153928 container start 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:31.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.989Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:38:31.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.989Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:38:31.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.989Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi055 (none))" 2024-04-26T04:38:31.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.989Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.989Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.990Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.990Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.992Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.992Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.994Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.994Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.541µs 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.994Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.994Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.994Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.859µs wal_replay_duration=189.078µs wbl_replay_duration=111ns total_replay_duration=225.649µs 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.996Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.996Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:38:31.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:30.996Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:38:31.109 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:31.013Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.34497ms db_storage=711ns remote_storage=1.069µs web_handler=259ns query_engine=579ns scrape=370.867µs scrape_sd=17.168µs notify=1.166µs notify_sd=2.274µs rules=16.804833ms tracing=5.768µs 2024-04-26T04:38:31.109 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:31.013Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:38:31.109 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:38:31.013Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:38:31.109 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:31 smithi055 bash[58193]: 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec 2024-04-26T04:38:31.109 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:38:31 smithi055 systemd[1]: Started Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:38:32.277 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-mon[30328]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ignoring --setuser ceph since I am not root 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ignoring --setgroup ceph since I am not root 2024-04-26T04:38:32.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-mgr[32179]: -- 172.21.15.55:0/3993066083 <== mon.1 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563ee28309c0 con 0x563ee2807000 2024-04-26T04:38:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:38:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[36743]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mon[26454]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setuser ceph since I am not root 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setgroup ceph since I am not root 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-mgr[26688]: -- 172.21.15.38:0/2358280864 <== mon.2 v2:172.21.15.38:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56094c9529c0 con 0x56094c928800 2024-04-26T04:38:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:32.327+0000 7f4305d311c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:38:32.599 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:32.327+0000 7fc0822a21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:38:32.599 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:32.496+0000 7fc0822a21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:32.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:32.604+0000 7fc0822a21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:38:32.917 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:32.494+0000 7f4305d311c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:32.917 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:32.598+0000 7f4305d311c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:38:32.917 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:32.916+0000 7f4305d311c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:33 smithi038 ceph-mon[26454]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:33 smithi038 ceph-mon[26454]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:33.001+0000 7f4305d311c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:33 smithi038 ceph-mon[36743]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:38:33.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:33 smithi038 ceph-mon[36743]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:38:33.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:33 smithi055 ceph-mon[30328]: from='mgr.14150 172.21.15.38:0/590456074' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-26T04:38:33.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:33 smithi055 ceph-mon[30328]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-26T04:38:33.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:32.924+0000 7fc0822a21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:38:33.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:33.011+0000 7fc0822a21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:38:34.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:34.211+0000 7f4305d311c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:38:34.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:34.383+0000 7f4305d311c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:34.557 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:34.213+0000 7fc0822a21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:38:34.557 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:34.383+0000 7fc0822a21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:34.558 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:34.555+0000 7fc0822a21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:38:34.830 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:34.556+0000 7f4305d311c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:38:35.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:34.827+0000 7fc0822a21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:38:35.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:34.920+0000 7fc0822a21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:38:35.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:35.017+0000 7fc0822a21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:38:35.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:34.829+0000 7f4305d311c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:38:35.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:34.924+0000 7f4305d311c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:38:35.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:35.022+0000 7f4305d311c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:38:35.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:35.301+0000 7fc0822a21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:35.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:35.309+0000 7f4305d311c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:38:36.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:35.779+0000 7fc0822a21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:38:36.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:35.793+0000 7f4305d311c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:38:36.378 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:36.329+0000 7fc0822a21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:38:36.629 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:36.343+0000 7f4305d311c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:38:36.629 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:36.441+0000 7f4305d311c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:38:36.629 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:36.628+0000 7f4305d311c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:38:36.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:36.430+0000 7fc0822a21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:38:36.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:36.620+0000 7fc0822a21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:38:37.197 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:36.929+0000 7fc0822a21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:38:37.203 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:36.935+0000 7f4305d311c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:38:37.450 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:37.195+0000 7fc0822a21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:38:37.454 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:37.202+0000 7f4305d311c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:38:37.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:37.453+0000 7f4305d311c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:38:37.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:37.591+0000 7f4305d311c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:38:37.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:37.690+0000 7f4305d311c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:38:37.801 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:37.448+0000 7fc0822a21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:38:37.801 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:37.585+0000 7fc0822a21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:38:37.801 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:37.681+0000 7fc0822a21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:38:37.801 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:37.799+0000 7fc0822a21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:38.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:37.892+0000 7fc0822a21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:38:38.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:37.812+0000 7f4305d311c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:38:38.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:37.906+0000 7f4305d311c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:38:38.974 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:38.716+0000 7fc0822a21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:38:38.974 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:38.915+0000 7fc0822a21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:38:38.974 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: [26/Apr/2024:04:38:38] ENGINE Bus STARTING 2024-04-26T04:38:39.032 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:38.745+0000 7f4305d311c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:38:39.032 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:38.947+0000 7f4305d311c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:38:39.032 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: Standby manager daemon x restarted 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: Standby manager daemon x started 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: Active manager daemon y restarted 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: Activating manager daemon y 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: mgrmap e17: y(active, starting, since 0.012636s), standbys: x 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:38:39.033 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: Standby manager daemon x restarted 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: Standby manager daemon x started 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: Active manager daemon y restarted 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: Activating manager daemon y 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: mgrmap e17: y(active, starting, since 0.012636s), standbys: x 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:38:39.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:38:39.035 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:38 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:38:39.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: CherryPy Checker: 2024-04-26T04:38:39.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: The Application mounted at '' has an empty config. 2024-04-26T04:38:39.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: 2024-04-26T04:38:39.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: [26/Apr/2024:04:38:39] ENGINE Serving on http://:::9283 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: [26/Apr/2024:04:38:39] ENGINE Bus STARTED 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: Standby manager daemon x restarted 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: Standby manager daemon x started 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2393952387' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: Active manager daemon y restarted 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: Activating manager daemon y 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: osdmap e60: 8 total, 8 up, 8 in 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: mgrmap e17: y(active, starting, since 0.012636s), standbys: x 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:38:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:38:39.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:38:39.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:38 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:38:39.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:38:39] ENGINE Bus STARTING 2024-04-26T04:38:39.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: CherryPy Checker: 2024-04-26T04:38:39.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: The Application mounted at '' has an empty config. 2024-04-26T04:38:39.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:38:39.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:38:39] ENGINE Serving on http://:::9283 2024-04-26T04:38:39.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:38:39] ENGINE Bus STARTED 2024-04-26T04:38:40.008 INFO:teuthology.orchestra.run.smithi055.stdout:Scheduled grafana update... 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:38:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: Manager daemon y is now available 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:38:40.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: Manager daemon y is now available 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:40.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:38:40.228 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:40.228 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:40.228 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:40.228 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:38:40.250 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: Manager daemon y is now available 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:40.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:38:40.822 DEBUG:teuthology.orchestra.run.smithi055:grafana.a> sudo journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service 2024-04-26T04:38:40.825 INFO:tasks.cephadm:Setting up client nodes... 2024-04-26T04:38:40.825 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:38:41.095 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: mgrmap e18: y(active, since 1.04246s), standbys: x 2024-04-26T04:38:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: Saving service grafana spec with placement smithi055=a;count:1 2024-04-26T04:38:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: [26/Apr/2024:04:38:40] ENGINE Bus STARTING 2024-04-26T04:38:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:41 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: mgrmap e18: y(active, since 1.04246s), standbys: x 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: Saving service grafana spec with placement smithi055=a;count:1 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: [26/Apr/2024:04:38:40] ENGINE Bus STARTING 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: mgrmap e18: y(active, since 1.04246s), standbys: x 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: Saving service grafana spec with placement smithi055=a;count:1 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: [26/Apr/2024:04:38:40] ENGINE Bus STARTING 2024-04-26T04:38:41.220 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.220 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:41.220 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:41 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.297 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: [26/Apr/2024:04:38:40] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: [26/Apr/2024:04:38:40] ENGINE Bus STARTED 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: [26/Apr/2024:04:38:40] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: [26/Apr/2024:04:38:40] ENGINE Bus STARTED 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:42.298 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.299 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.299 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:38:42.299 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.299 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:42 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: [26/Apr/2024:04:38:40] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: [26/Apr/2024:04:38:40] ENGINE Bus STARTED 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-26T04:38:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:42 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.125 INFO:teuthology.orchestra.run.smithi038.stdout:[client.0] 2024-04-26T04:38:43.125 INFO:teuthology.orchestra.run.smithi038.stdout: key = AQDTLytmbLwIBxAAWjNQ19EKtRf3PrpPENlX+Q== 2024-04-26T04:38:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi055 to 4017M 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:38:43.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2368705427' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi055 to 4017M 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2368705427' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:38:43.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:43.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:43 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:43.909 DEBUG:teuthology.orchestra.run.smithi038:> set -ex 2024-04-26T04:38:43.910 DEBUG:teuthology.orchestra.run.smithi038:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-26T04:38:43.910 DEBUG:teuthology.orchestra.run.smithi038:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-26T04:38:43.948 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-26T04:38:43.970 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.970 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.970 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi055 to 4017M 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:38:43.971 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:38:43.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:43.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2368705427' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:43.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:43.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:38:43.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:43.972 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:43 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:38:44.208 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.b/config 2024-04-26T04:38:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:38:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:38:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:38:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[26454]: Deploying daemon alertmanager.a on smithi038 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:44.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:44 smithi038 ceph-mon[36743]: Deploying daemon alertmanager.a on smithi038 2024-04-26T04:38:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:38:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:38:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-26T04:38:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:44 smithi055 ceph-mon[30328]: Deploying daemon alertmanager.a on smithi038 2024-04-26T04:38:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:45 smithi055 ceph-mon[30328]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:45.946 INFO:teuthology.orchestra.run.smithi055.stdout:[client.1] 2024-04-26T04:38:45.946 INFO:teuthology.orchestra.run.smithi055.stdout: key = AQDVLytmcXjhNxAAp/pDZT8e4iQKCBbW4hSATA== 2024-04-26T04:38:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:45 smithi038 ceph-mon[26454]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:45.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:45 smithi038 ceph-mon[36743]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:46.596 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2024-04-26T04:38:46.596 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-26T04:38:46.596 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-26T04:38:46.660 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:38:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:38:46.676 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-26T04:38:46.676 DEBUG:teuthology.orchestra.run.smithi038:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-26T04:38:46.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:46 smithi038 ceph-mon[26454]: from='client.? 172.21.15.55:0/417977547' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:46.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:46 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:46.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:46 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:38:46.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:46 smithi038 ceph-mon[36743]: from='client.? 172.21.15.55:0/417977547' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:46.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:46 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:46.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:46 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:38:46.999 INFO:teuthology.misc.health.smithi038.stdout: 2024-04-26T04:38:46.999 INFO:teuthology.misc.health.smithi038.stdout:{"epoch":60,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:38:38.949654+0000","last_up_change":"2024-04-26T04:38:07.872498+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6803","nonce":3619308679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6805","nonce":3619308679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6809","nonce":3619308679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6807","nonce":3619308679}]},"public_addr":"172.21.15.38:6803/3619308679","cluster_addr":"172.21.15.38:6805/3619308679","heartbeat_back_addr":"172.21.15.38:6809/3619308679","heartbeat_front_addr":"172.21.15.38:6807/3619308679","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6811","nonce":2181836257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6813","nonce":2181836257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6817","nonce":2181836257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6815","nonce":2181836257}]},"public_addr":"172.21.15.38:6811/2181836257","cluster_addr":"172.21.15.38:6813/2181836257","heartbeat_back_addr":"172.21.15.38:6817/2181836257","heartbeat_front_addr":"172.21.15.38:6815/2181836257","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6819","nonce":124975504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6821","nonce":124975504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6825","nonce":124975504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6823","nonce":124975504}]},"public_addr":"172.21.15.38:6819/124975504","cluster_addr":"172.21.15.38:6821/124975504","heartbeat_back_addr":"172.21.15.38:6825/124975504","heartbeat_front_addr":"172.21.15.38:6823/124975504","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6827","nonce":1698863327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6829","nonce":1698863327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6833","nonce":1698863327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6831","nonce":1698863327}]},"public_addr":"172.21.15.38:6827/1698863327","cluster_addr":"172.21.15.38:6829/1698863327","heartbeat_back_addr":"172.21.15.38:6833/1698863327","heartbeat_front_addr":"172.21.15.38:6831/1698863327","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:38:47.010 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-26T04:38:47.010 INFO:tasks.cephadm:Creating RBD pool 2024-04-26T04:38:47.010 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-26T04:38:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:46 smithi055 ceph-mon[30328]: from='client.? 172.21.15.55:0/417977547' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:46 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-26T04:38:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:46 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-26T04:38:47.645 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:47.768 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:47 smithi038 ceph-mon[26454]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:38:47.768 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:47 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3692336246' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:38:47.768 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:47 smithi038 ceph-mon[36743]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:38:47.768 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:47 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3692336246' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:38:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:47 smithi055 ceph-mon[30328]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-26T04:38:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:47 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3692336246' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:38:50.211 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:50 smithi038 ceph-mon[26454]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:38:50.211 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:50 smithi038 ceph-mon[36743]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:38:50.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:50 smithi055 ceph-mon[30328]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-26T04:38:51.560 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:51 smithi038 systemd[1]: Starting Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:38:52.172 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-mon[26454]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:38:52.172 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-mon[36743]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:38:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:52 smithi055 ceph-mon[30328]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-26T04:38:52.465 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 podman[65958]: 2024-04-26 04:38:52.171693658 +0000 UTC m=+0.329162793 volume create 1d6212a4ac4235d8ff16fbba37fd4f1a304faf4377e937489995858a7135cbb1 2024-04-26T04:38:52.465 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 podman[65958]: 2024-04-26 04:38:52.093615822 +0000 UTC m=+0.251084976 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:38:52.465 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 podman[65958]: 2024-04-26 04:38:52.272261962 +0000 UTC m=+0.429731150 container create 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:52.974 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 podman[65958]: 2024-04-26 04:38:52.591325584 +0000 UTC m=+0.748794718 container init 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:52.974 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 podman[65958]: 2024-04-26 04:38:52.594003543 +0000 UTC m=+0.751472680 container start 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:52.974 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.606Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:38:52.974 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.606Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:38:52.974 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.607Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.38 port=9094 2024-04-26T04:38:52.974 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.608Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:38:52.975 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.640Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:38:52.975 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.640Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:38:52.975 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.643Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:38:52.975 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:52.643Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:38:52.975 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 bash[65958]: 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 2024-04-26T04:38:52.975 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:52 smithi038 systemd[1]: Started Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:38:53.067 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'rbd' created 2024-04-26T04:38:53.312 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3819720424' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3819720424' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.313 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:53 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:53 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3819720424' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:38:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:53 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-26T04:38:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:53 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:53 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:53 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:53 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:53.614 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-26T04:38:53.904 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:54.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[26454]: Deploying daemon node-exporter.a on smithi038 2024-04-26T04:38:54.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[26454]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:38:54.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:38:54.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[26454]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:38:54.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:38:54.175 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:54.176 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[36743]: Deploying daemon node-exporter.a on smithi038 2024-04-26T04:38:54.176 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[36743]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:38:54.176 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:38:54.176 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[36743]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:38:54.176 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:38:54.176 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:54.176 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:54 smithi038 systemd[1]: Starting Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:38:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:54 smithi055 ceph-mon[30328]: Deploying daemon node-exporter.a on smithi038 2024-04-26T04:38:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:54 smithi055 ceph-mon[30328]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-26T04:38:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:54 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-26T04:38:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:54 smithi055 ceph-mon[30328]: osdmap e61: 8 total, 8 up, 8 in 2024-04-26T04:38:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:54 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:38:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:54 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:54.879 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:38:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:38:54.609Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000442853s 2024-04-26T04:38:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:55 smithi038 ceph-mon[26454]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:38:55.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:55 smithi038 ceph-mon[36743]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:38:55.224 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:55 smithi038 bash[66366]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:38:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:55 smithi055 ceph-mon[30328]: osdmap e62: 8 total, 8 up, 8 in 2024-04-26T04:38:55.724 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:55 smithi038 bash[66366]: Getting image source signatures 2024-04-26T04:38:55.724 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:55 smithi038 bash[66366]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:38:55.724 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:55 smithi038 bash[66366]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:38:55.724 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:55 smithi038 bash[66366]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:38:56.088 INFO:teuthology.orchestra.run.smithi038.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-26T04:38:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:56 smithi055 ceph-mon[30328]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:56 smithi055 ceph-mon[30328]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:38:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:56 smithi055 ceph-mon[30328]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:38:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:56 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3669473164' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:38:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[26454]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[26454]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:38:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[26454]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:38:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3669473164' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:38:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[36743]: pgmap v13: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[36743]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:38:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[36743]: osdmap e63: 8 total, 8 up, 8 in 2024-04-26T04:38:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:56 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3669473164' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:38:57.092 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:38:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:38:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:38:57.243 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:38:57.243 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:38:57.243 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:38:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:57 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3669473164' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:38:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:57 smithi055 ceph-mon[30328]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:38:57.437 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3669473164' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:38:57.437 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-mon[26454]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:38:57.437 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3669473164' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:38:57.437 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-mon[36743]: osdmap e64: 8 total, 8 up, 8 in 2024-04-26T04:38:57.437 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 bash[66366]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:38:57.438 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 bash[66366]: Writing manifest to image destination 2024-04-26T04:38:58.018 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 podman[66366]: 2024-04-26 04:38:57.70865076 +0000 UTC m=+2.787597844 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:38:58.018 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 podman[66366]: 2024-04-26 04:38:57.807182873 +0000 UTC m=+2.886129988 container create b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:58.018 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 podman[66366]: 2024-04-26 04:38:57.950866422 +0000 UTC m=+3.029813498 container init b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 podman[66366]: 2024-04-26 04:38:57.95365421 +0000 UTC m=+3.032601290 container start b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.957Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.957Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.957Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.957Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:38:58.019 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:38:58.020 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:38:58.021 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:38:58.022 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.958Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[66536]: ts=2024-04-26T04:38:57.959Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:38:58.023 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:58 smithi038 bash[66366]: b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 2024-04-26T04:38:58.084 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:38:58.330 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[26454]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:58.330 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[26454]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:38:58.330 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[26454]: Cluster is now healthy 2024-04-26T04:38:58.330 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:58.331 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[36743]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:58.331 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[36743]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:38:58.331 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[36743]: Cluster is now healthy 2024-04-26T04:38:58.331 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:58 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:58.331 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:38:58 smithi038 systemd[1]: Started Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:38:58.400 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:58 smithi055 ceph-mon[30328]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:38:58.400 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:58 smithi055 ceph-mon[30328]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:38:58.400 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:58 smithi055 ceph-mon[30328]: Cluster is now healthy 2024-04-26T04:38:58.400 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:58 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.053 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:38:58 smithi055 systemd[1]: Starting Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:38:59.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:59 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:59 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:38:59 smithi038 ceph-mon[26454]: Deploying daemon node-exporter.b on smithi055 2024-04-26T04:38:59.192 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:59 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.192 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:59 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.192 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:38:59 smithi038 ceph-mon[36743]: Deploying daemon node-exporter.b on smithi055 2024-04-26T04:38:59.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:59 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:59 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:38:59.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:38:59 smithi055 ceph-mon[30328]: Deploying daemon node-exporter.b on smithi055 2024-04-26T04:38:59.357 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:38:59 smithi055 bash[60288]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-26T04:38:59.856 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:38:59 smithi055 bash[60288]: Getting image source signatures 2024-04-26T04:38:59.856 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:38:59 smithi055 bash[60288]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-26T04:38:59.856 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:38:59 smithi055 bash[60288]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-26T04:38:59.856 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:38:59 smithi055 bash[60288]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-26T04:39:00.014 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:39:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:00 smithi038 ceph-mon[26454]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:00 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3290454227' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:39:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:00 smithi038 ceph-mon[36743]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:00 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3290454227' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:39:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:00 smithi055 ceph-mon[30328]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:00 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3290454227' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:39:01.181 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":20,"flags":0,"active_gid":24332,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6800","nonce":3488049706},{"type":"v1","addr":"172.21.15.38:6801","nonce":3488049706}]},"active_addr":"172.21.15.38:6801/3488049706","active_change":"2024-04-26T04:38:38.949817+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.38:8443/","prometheus":"http://172.21.15.38:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":3149555040}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":1564621640}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":4062968598}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":1699336096}]}]} 2024-04-26T04:39:01.185 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:39:01.185 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:39:01.185 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:39:01.278 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:00 smithi055 bash[60288]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-26T04:39:01.278 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:00 smithi055 bash[60288]: Writing manifest to image destination 2024-04-26T04:39:01.452 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:01.544 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 podman[60288]: 2024-04-26 04:39:01.240098135 +0000 UTC m=+2.286047950 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:39:01.544 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 podman[60288]: 2024-04-26 04:39:01.325010728 +0000 UTC m=+2.370960587 container create 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 podman[60288]: 2024-04-26 04:39:01.476966937 +0000 UTC m=+2.522916751 container init 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 podman[60288]: 2024-04-26 04:39:01.480185651 +0000 UTC m=+2.526135470 container start 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.484Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.484Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.485Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.485Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:39:01.545 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:39:01.546 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:39:01.546 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:39:01.546 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:39:01.546 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:39:01.546 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:39:01.546 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:39:01.547 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:39:01.548 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:39:01.549 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:39:01.550 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.486Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:39:01.550 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.487Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:39:01.550 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[60337]: ts=2024-04-26T04:39:01.487Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:39:01.550 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 bash[60288]: 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 2024-04-26T04:39:01.846 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:01] ENGINE Bus STOPPING 2024-04-26T04:39:01.846 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:39:01.846 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:01] ENGINE Bus STOPPED 2024-04-26T04:39:01.846 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:01] ENGINE Bus STARTING 2024-04-26T04:39:01.856 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:39:01 smithi055 systemd[1]: Started Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:39:02.148 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:01] ENGINE Serving on http://:::9283 2024-04-26T04:39:02.148 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:01] ENGINE Bus STARTED 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:02.840 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:02 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:39:02.612Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003649415s 2024-04-26T04:39:03.284 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:39:03.284 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":64,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:38:56.066375+0000","last_up_change":"2024-04-26T04:38:07.872498+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:38:52.695557+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6803","nonce":3619308679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6805","nonce":3619308679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6809","nonce":3619308679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6807","nonce":3619308679}]},"public_addr":"172.21.15.38:6803/3619308679","cluster_addr":"172.21.15.38:6805/3619308679","heartbeat_back_addr":"172.21.15.38:6809/3619308679","heartbeat_front_addr":"172.21.15.38:6807/3619308679","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6811","nonce":2181836257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6813","nonce":2181836257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6817","nonce":2181836257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6815","nonce":2181836257}]},"public_addr":"172.21.15.38:6811/2181836257","cluster_addr":"172.21.15.38:6813/2181836257","heartbeat_back_addr":"172.21.15.38:6817/2181836257","heartbeat_front_addr":"172.21.15.38:6815/2181836257","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6819","nonce":124975504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6821","nonce":124975504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6825","nonce":124975504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6823","nonce":124975504}]},"public_addr":"172.21.15.38:6819/124975504","cluster_addr":"172.21.15.38:6821/124975504","heartbeat_back_addr":"172.21.15.38:6825/124975504","heartbeat_front_addr":"172.21.15.38:6823/124975504","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6827","nonce":1698863327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6829","nonce":1698863327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6833","nonce":1698863327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6831","nonce":1698863327}]},"public_addr":"172.21.15.38:6827/1698863327","cluster_addr":"172.21.15.38:6829/1698863327","heartbeat_back_addr":"172.21.15.38:6833/1698863327","heartbeat_front_addr":"172.21.15.38:6831/1698863327","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:39:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:03 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:03 smithi038 ceph-mon[26454]: Deploying daemon grafana.a on smithi055 2024-04-26T04:39:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:03 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:03.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:03 smithi038 ceph-mon[36743]: Deploying daemon grafana.a on smithi055 2024-04-26T04:39:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:03 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-26T04:39:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:03 smithi055 ceph-mon[30328]: Deploying daemon grafana.a on smithi055 2024-04-26T04:39:03.976 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:39:03.976 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:39:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:04 smithi038 ceph-mon[26454]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:04 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/532702470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:04 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:04 smithi038 ceph-mon[36743]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:04 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/532702470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:04 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:04.242 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:04 smithi055 ceph-mon[30328]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:04 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/532702470' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:04 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:05.841 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:39:05.841 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":64,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:38:56.066375+0000","last_up_change":"2024-04-26T04:38:07.872498+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:38:52.695557+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6803","nonce":3619308679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6805","nonce":3619308679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6809","nonce":3619308679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6807","nonce":3619308679}]},"public_addr":"172.21.15.38:6803/3619308679","cluster_addr":"172.21.15.38:6805/3619308679","heartbeat_back_addr":"172.21.15.38:6809/3619308679","heartbeat_front_addr":"172.21.15.38:6807/3619308679","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6811","nonce":2181836257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6813","nonce":2181836257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6817","nonce":2181836257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6815","nonce":2181836257}]},"public_addr":"172.21.15.38:6811/2181836257","cluster_addr":"172.21.15.38:6813/2181836257","heartbeat_back_addr":"172.21.15.38:6817/2181836257","heartbeat_front_addr":"172.21.15.38:6815/2181836257","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6819","nonce":124975504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6821","nonce":124975504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6825","nonce":124975504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6823","nonce":124975504}]},"public_addr":"172.21.15.38:6819/124975504","cluster_addr":"172.21.15.38:6821/124975504","heartbeat_back_addr":"172.21.15.38:6825/124975504","heartbeat_front_addr":"172.21.15.38:6823/124975504","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6827","nonce":1698863327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6829","nonce":1698863327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6833","nonce":1698863327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6831","nonce":1698863327}]},"public_addr":"172.21.15.38:6827/1698863327","cluster_addr":"172.21.15.38:6829/1698863327","heartbeat_back_addr":"172.21.15.38:6833/1698863327","heartbeat_front_addr":"172.21.15.38:6831/1698863327","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:39:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:06 smithi038 ceph-mon[26454]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:06 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1274212601' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:06 smithi038 ceph-mon[36743]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:06 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1274212601' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.521 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:39:06.521 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:39:06.522 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:39:06.522 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:39:06.522 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:39:06.523 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:39:06.523 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:39:06.524 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:39:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:06 smithi055 ceph-mon[30328]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:06 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1274212601' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:39:06.899 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:39:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:39:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:07.091 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.091 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.094 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.095 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.096 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.096 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.097 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:07.126 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:08 smithi038 ceph-mon[26454]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:08 smithi038 ceph-mon[36743]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:08.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:08 smithi055 ceph-mon[30328]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:09.452 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:09.453 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:10 smithi038 ceph-mon[26454]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:10 smithi038 ceph-mon[36743]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:10 smithi055 ceph-mon[30328]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:12.473 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:12 smithi038 ceph-mon[26454]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:12 smithi038 ceph-mon[36743]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:12 smithi055 ceph-mon[30328]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:13.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:13 smithi038 ceph-mon[26454]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:13.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:13 smithi038 ceph-mon[36743]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:13.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:13 smithi055 ceph-mon[30328]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:13.652 INFO:teuthology.orchestra.run.smithi038.stdout:236223201294 2024-04-26T04:39:13.652 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:39:15.817 INFO:teuthology.orchestra.run.smithi038.stdout:77309411368 2024-04-26T04:39:15.817 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:39:16.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:16 smithi038 ceph-mon[26454]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:16.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:16 smithi038 ceph-mon[36743]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:16 smithi055 ceph-mon[30328]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:17.095 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:39:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:39:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:18.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:18 smithi055 ceph-mon[30328]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:18.264 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:18 smithi038 ceph-mon[36743]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:18.264 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:18 smithi038 ceph-mon[26454]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:19.050 INFO:teuthology.orchestra.run.smithi038.stdout:133143986206 2024-04-26T04:39:19.051 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:39:19.830 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:19 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:39:20.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-mon[30328]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:20.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:19 smithi055 podman[60680]: 2024-04-26 04:39:19.728717683 +0000 UTC m=+0.017994902 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:39:20.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:19 smithi055 podman[60680]: 2024-04-26 04:39:19.839275507 +0000 UTC m=+0.128552704 container create 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:39:20.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 podman[60680]: 2024-04-26 04:39:20.041884321 +0000 UTC m=+0.331161519 container init 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , vendor=Red Hat, Inc., description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.buildah.version=1.32.0, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, distribution-scope=public, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:39:20.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 podman[60680]: 2024-04-26 04:39:20.044450742 +0000 UTC m=+0.333727941 container start 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, name=ubi8, architecture=x86_64, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , io.buildah.version=1.32.0, version=8.5, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8) 2024-04-26T04:39:20.158 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:20 smithi038 ceph-mon[36743]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:20.158 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:20 smithi038 ceph-mon[26454]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:20.357 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 bash[60680]: 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.22030158Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220543156Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220568845Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220581237Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220592959Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220603519Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220615429Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-26T04:39:20.358 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220626135Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220636307Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220646443Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220656367Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220666444Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220675975Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=settings t=2024-04-26T04:39:20.220686255Z level=info msg="App mode production" 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=sqlstore t=2024-04-26T04:39:20.220750293Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=sqlstore t=2024-04-26T04:39:20.220776206Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.222008911Z level=info msg="Starting DB migrations" 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.222113377Z level=info msg="Executing migration" id="create migration_log table" 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.222876915Z level=info msg="Executing migration" id="create user table" 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.223383808Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-26T04:39:20.359 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.223953476Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.228957585Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.233961625Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.236253512Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.237414942Z level=info msg="Executing migration" id="create user table v2" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.240023335Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.24234612Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.242981687Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.245606522Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.248251935Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.250629591Z level=info msg="Executing migration" id="Update user table charset" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.253013897Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.255234048Z level=info msg="Executing migration" id="Add missing user data" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.25577598Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-26T04:39:20.360 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.258462066Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-26T04:39:20.361 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.260773929Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-26T04:39:20.361 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.26127457Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-26T04:39:20.362 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.262483554Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-26T04:39:20.362 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.262971093Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-26T04:39:20.362 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.263442129Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-26T04:39:20.362 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.263876293Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.264306799Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.264720116Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.265045301Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.265457001Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.265837823Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.266274134Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.26683083Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.267754848Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.26831555Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.268849082Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.269354297Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-26T04:39:20.363 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.269807692Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.270301314Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.270734863Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.271171586Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.271582441Z level=info msg="Executing migration" id="create star table" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.272066813Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.272773695Z level=info msg="Executing migration" id="create org table v1" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.273445076Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.273978063Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.274455476Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.274958049Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.275483126Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.275976849Z level=info msg="Executing migration" id="Update org table charset" 2024-04-26T04:39:20.364 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.27637959Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.276743368Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.277154435Z level=info msg="Executing migration" id="create dashboard table" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.277640831Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.278169748Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.278685822Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.279181932Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.279703903Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.280223304Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.281191775Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.281680067Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.28221438Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.282727305Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.283197638Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.283683093Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-26T04:39:20.365 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.284045402Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.284629429Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.285223602Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.28579997Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.286377567Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.2869998Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.287532118Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.287999012Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.288368992Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.288737359Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.289336737Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.289924484Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-26T04:39:20.366 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.290516937Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.291070967Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.291486736Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.291994169Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.292502266Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.29285541Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.29340878Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.293884516Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.294915372Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.295433517Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.295983168Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.296534942Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.296956811Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-26T04:39:20.367 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.297447129Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.298032277Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.2985845Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.298979031Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.299441896Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.299990912Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.300648304Z level=info msg="Executing migration" id="create data_source table" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.301193424Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.301663255Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.306952633Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.312020406Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.312580351Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.313734012Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-26T04:39:20.368 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.314309669Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.314844575Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.315410871Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.315846811Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.316507986Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.317143841Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.317574525Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.317974934Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.318622231Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.319040115Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.319479687Z level=info msg="Executing migration" id="Add uid column" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.320092308Z level=info msg="Executing migration" id="Update uid value" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.32054966Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.321069473Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-26T04:39:20.369 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.32162561Z level=info msg="Executing migration" id="create api_key table" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.322173371Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.322724292Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.32328824Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.323853229Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.324419009Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.324928612Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.325445026Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.326545713Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.326994411Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.327490412Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.328002701Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-26T04:39:20.370 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.328521723Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.328874696Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.329409225Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.329923848Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.330530227Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.33112795Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.331482611Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.332064753Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.332688161Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.333144048Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.333574862Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.333989774Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.334489583Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.334947918Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.335424866Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-26T04:39:20.371 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.335726983Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.336048168Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.336645755Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.337258052Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.337598205Z level=info msg="Executing migration" id="create quota table v1" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.338019077Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.338532149Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.338879776Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.339373148Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.339861701Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.340500906Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.340823378Z level=info msg="Executing migration" id="create session table" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.341319644Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.341670705Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.342007047Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.342519137Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.343013141Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-26T04:39:20.372 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.343421559Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.343795893Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.344218089Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.344669082Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.3451999Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.345608186Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.346536448Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.34700283Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.347727926Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.34844983Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.348851524Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.349598057Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.350175057Z level=info msg="Executing migration" id="create alert table v1" 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.35074475Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-26T04:39:20.373 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.351317028Z level=info msg="Executing migration" id="add index alert state" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.351842281Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.352381126Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.352857147Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.353412732Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.353894889Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.355795257Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.356316079Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.357013987Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.357485698Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.357988569Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-26T04:39:20.374 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.358524947Z level=info msg="Executing migration" id="Add column is_default" 2024-04-26T04:39:20.375 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.359427644Z level=info msg="Executing migration" id="Add column frequency" 2024-04-26T04:39:20.375 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.360241747Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-26T04:39:20.375 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.361028881Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-26T04:39:20.375 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.361855354Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-26T04:39:20.674 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.36241906Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-26T04:39:20.674 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.36278825Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-26T04:39:20.674 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.363271803Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-26T04:39:20.674 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.363811765Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-26T04:39:20.674 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.364439983Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-26T04:39:20.675 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.365008319Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-26T04:39:20.675 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.365572868Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-26T04:39:20.675 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.366174578Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-26T04:39:20.675 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.367059083Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-26T04:39:20.675 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.367875277Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.368333496Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.368975436Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.369499487Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.370271501Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.370691607Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.371237515Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.371768746Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-26T04:39:20.676 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.37220758Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.372760175Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.373372909Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.373971069Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.374553898Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.375129366Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.375658655Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.376060709Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.376825316Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-26T04:39:20.677 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.377321628Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.378191133Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.378673748Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.379187581Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.379607807Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.381231817Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.381636085Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.382090672Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-26T04:39:20.678 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.382427004Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.382775505Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.383116709Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.383833879Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.384524523Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.384952269Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.385387446Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-26T04:39:20.679 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.385693824Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.386403482Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.386850492Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.387198807Z level=info msg="Executing migration" id="Move region to single row" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.387649931Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.388103986Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.388532661Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.388948027Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-26T04:39:20.680 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.389561989Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-26T04:39:20.681 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.389971864Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-26T04:39:20.681 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.39040936Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-26T04:39:20.681 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.390720721Z level=info msg="Executing migration" id="create test_data table" 2024-04-26T04:39:20.681 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.391186163Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-26T04:39:20.681 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.391605586Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-26T04:39:20.681 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.392050067Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-26T04:39:20.683 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.392501321Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-26T04:39:20.683 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.392814274Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-26T04:39:20.683 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.393183716Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-26T04:39:20.683 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.393529504Z level=info msg="Executing migration" id="create team table" 2024-04-26T04:39:20.683 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.394000776Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-26T04:39:20.683 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.394686467Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.395245969Z level=info msg="Executing migration" id="create team member table" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.397842126Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.400294601Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.400984694Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.401505497Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.404203003Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.406589363Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.407534324Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.408069403Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.410348384Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.412934964Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.413563436Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.414129828Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.414639239Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-26T04:39:20.684 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.415153996Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.415659559Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.416283645Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.416684832Z level=info msg="Executing migration" id="create tag table" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.417187633Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.417709231Z level=info msg="Executing migration" id="create login attempt table" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.418210548Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.418732559Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.419265571Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.421297758Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.421786755Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.42232954Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.422750439Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.423230678Z level=info msg="Executing migration" id="create user auth table" 2024-04-26T04:39:20.685 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.423714118Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.424260324Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.424671815Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.425606277Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.426483727Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.427372505Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.428271448Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.428841246Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.429715544Z level=info msg="Executing migration" id="create server_lock table" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.430243711Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.430805616Z level=info msg="Executing migration" id="create user auth token table" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.431368969Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.431958539Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-26T04:39:20.686 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.43250379Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.433036993Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.433936955Z level=info msg="Executing migration" id="create cache_data table" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.434466786Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.435020072Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.435549994Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.436070779Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.43647781Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.436864905Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.437495689Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.438032623Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-26T04:39:20.687 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.438574532Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.438980291Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.439510303Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.440010427Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.440542865Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.441060457Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.441961457Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.442494628Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.442879444Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.4434131Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.443921201Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.444436647Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.444796829Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.445370689Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-26T04:39:20.688 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.445868887Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.446405178Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.446994525Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.447918904Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.448464959Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.448961944Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.452522638Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.456061239Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.45661379Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.457153374Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.458107473Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.458647087Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.459220928Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.459777826Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-26T04:39:20.689 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.460325992Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.460714431Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.461650583Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.462595137Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.463504837Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.464013875Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.464554271Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.465480039Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.466425433Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.467025185Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.467946097Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.468847549Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.469218932Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.469699425Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-26T04:39:20.690 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.470205873Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.470726955Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.471086119Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.472017773Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.472893164Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.473767739Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.474647795Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.475572748Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.475926291Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.476364467Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.477245065Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.477586503Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.478560936Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.479043898Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-26T04:39:20.691 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.479936066Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.480395353Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.480870804Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.481797614Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.482251436Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.482729607Z level=info msg="Executing migration" id="create alert_image table" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.483192838Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.483656524Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.483963246Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.48438863Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.484865125Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.484952193Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.485321827Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.485767562Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-26T04:39:20.692 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.486241322Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.486365468Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.486477002Z level=info msg="alerts found to migrate" alerts=0 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.486896714Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.48741929Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.487962855Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.488394062Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.488864816Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.489361317Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.489651833Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.490025916Z level=info msg="Executing migration" id="create data_keys table" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.490484789Z level=info msg="Executing migration" id="create secrets table" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.490937576Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.495392614Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-26T04:39:20.693 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.496353117Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.49669815Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.501055065Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.505458194Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.505933876Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.506433379Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.506804386Z level=info msg="Executing migration" id="create permission table" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.507261541Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.511594862Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.516538649Z level=info msg="Executing migration" id="create role table" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.51717061Z level=info msg="Executing migration" id="add column display_name" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.518754802Z level=info msg="Executing migration" id="add column group_name" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.520289131Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-26T04:39:20.694 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.520961052Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.5216221Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.522296937Z level=info msg="Executing migration" id="create team role table" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.522887939Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.523560977Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.524456695Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.525120412Z level=info msg="Executing migration" id="create user role table" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.525709095Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.526297572Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.526843083Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.527397394Z level=info msg="Executing migration" id="create builtin role table" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.527922218Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.528457084Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.529001714Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.530118041Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-26T04:39:20.695 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.5307248Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.531272242Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.531816596Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.53240193Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.532926932Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.533481267Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.53460735Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.535110907Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.535660826Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.536081106Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.536601009Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.537195795Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.537656229Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.538030819Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.538496391Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.538998347Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-26T04:39:20.696 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.539542995Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.540599767Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.541000056Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.541504416Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.542027239Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.542560368Z level=info msg="Executing migration" id="add correlation config column" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.543648879Z level=info msg="Executing migration" id="create entity_events table" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.544199518Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.544704876Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.544796747Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.545216632Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.545380837Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.545877718Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.546603236Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.547255019Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-26T04:39:20.697 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.547928311Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.548612027Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.549280703Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.549938523Z level=info msg="Executing migration" id="Drop public config table" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.55059554Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.551280101Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.551968826Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.552712391Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.553426369Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.559003442Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.560686491Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.562340821Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.562868388Z level=info msg="Executing migration" id="add share column" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.564586028Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-26T04:39:20.698 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.565357872Z level=info msg="Executing migration" id="create file table" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.566081262Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.566793765Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.567573435Z level=info msg="Executing migration" id="create file_meta table" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.568207378Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.568936648Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.569453352Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.570019662Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.570562756Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.571300967Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.573069922Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.573628636Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.57434089Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.574931071Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.575437378Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-26T04:39:20.699 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.575969588Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.577689608Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.579430154Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.580119377Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.592028831Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.592518606Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.593004803Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.596645122Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.596984264Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.597361819Z level=info msg="Executing migration" id="create folder table" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.597755986Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.598219514Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.59867284Z level=info msg="Executing migration" id="Update folder title length" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.59896602Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=migrator t=2024-04-26T04:39:20.599410005Z level=info msg="migrations completed" performed=485 skipped=0 duration=377.323994ms 2024-04-26T04:39:20.700 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=sqlstore t=2024-04-26T04:39:20.599824296Z level=info msg="Created default organization" 2024-04-26T04:39:20.701 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=plugin.loader t=2024-04-26T04:39:20.672124258Z level=info msg="Plugin registered" pluginID=input 2024-04-26T04:39:21.065 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=plugin.loader t=2024-04-26T04:39:20.705873445Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-26T04:39:21.065 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=plugin.loader t=2024-04-26T04:39:20.705921016Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-26T04:39:21.065 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=secrets t=2024-04-26T04:39:20.705980643Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-26T04:39:21.065 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=query_data t=2024-04-26T04:39:20.70797002Z level=info msg="Query Service initialization" 2024-04-26T04:39:21.065 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=live.push_http t=2024-04-26T04:39:20.710993605Z level=info msg="Live Push Gateway initialization" 2024-04-26T04:39:21.103 INFO:teuthology.orchestra.run.smithi038.stdout:167503724569 2024-04-26T04:39:21.103 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:39:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-mon[30328]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:21.405 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[36743]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.406 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:21.407 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:21.407 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:21 smithi038 ceph-mon[26454]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:21.430 INFO:teuthology.orchestra.run.smithi038.stdout:55834574892 2024-04-26T04:39:21.430 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:39:21.773 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=infra.usagestats.collector t=2024-04-26T04:39:21.506329749Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-26T04:39:21.773 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=provisioning.datasources t=2024-04-26T04:39:21.506796351Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-26T04:39:21.773 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=provisioning.datasources t=2024-04-26T04:39:21.540004179Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-26T04:39:21.773 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=provisioning.datasources t=2024-04-26T04:39:21.573981525Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=provisioning.alerting t=2024-04-26T04:39:21.603555486Z level=info msg="starting to provision alerting" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=provisioning.alerting t=2024-04-26T04:39:21.60358222Z level=info msg="finished to provision alerting" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=ngalert.state.manager t=2024-04-26T04:39:21.603719438Z level=info msg="Warming state cache for startup" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=grafanaStorageLogger t=2024-04-26T04:39:21.604109002Z level=info msg="storage starting" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=http.server t=2024-04-26T04:39:21.606520678Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=ngalert.state.manager t=2024-04-26T04:39:21.614972975Z level=info msg="State cache has been initialized" states=0 duration=11.246476ms 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=ticker t=2024-04-26T04:39:21.615317106Z level=info msg=starting first_tick=2024-04-26T04:39:30Z 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=ngalert.multiorg.alertmanager t=2024-04-26T04:39:21.615357786Z level=info msg="starting MultiOrg Alertmanager" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=sqlstore.transactions t=2024-04-26T04:39:21.666116911Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=sqlstore.transactions t=2024-04-26T04:39:21.677367034Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-26T04:39:21.774 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:39:21 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[60695]: logger=sqlstore.transactions t=2024-04-26T04:39:21.71567839Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-26T04:39:21.979 INFO:teuthology.orchestra.run.smithi038.stdout:201863462932 2024-04-26T04:39:21.979 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:39:22.202 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:23.281 INFO:teuthology.orchestra.run.smithi038.stdout:111669149732 2024-04-26T04:39:23.281 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:39:23.702 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:23.702 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:23.709 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:24.105 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:24 smithi038 ceph-mon[26454]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:24.105 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:24 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:24.105 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:24 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:24.106 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:24 smithi038 ceph-mon[36743]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:24.106 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:24 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:24.106 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:24 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:24.107 INFO:teuthology.orchestra.run.smithi038.stdout:34359738419 2024-04-26T04:39:24.107 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:39:24.283 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:24 smithi055 ceph-mon[30328]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:24 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:24.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:24 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:24.674 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:24.685 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:25.121 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:25 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:25.122 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:25 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:25.122 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:25 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:25.122 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:25 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:25.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:25 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:25.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:25 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:26.154 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:26 smithi038 ceph-mon[26454]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:26.154 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:26 smithi038 ceph-mon[36743]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:26.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:26 smithi055 ceph-mon[30328]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:27.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:39:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:39:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:27.397 INFO:teuthology.orchestra.run.smithi038.stdout:77309411370 2024-04-26T04:39:28.316 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:28 smithi038 ceph-mon[26454]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:28.317 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:28 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2668640335' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:39:28.317 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:28 smithi038 ceph-mon[36743]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:28.317 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:28 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2668640335' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:39:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:28 smithi055 ceph-mon[30328]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:28 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2668640335' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:39:30.006 INFO:teuthology.orchestra.run.smithi038.stdout:55834574895 2024-04-26T04:39:30.114 INFO:teuthology.orchestra.run.smithi038.stdout:167503724570 2024-04-26T04:39:30.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:30 smithi055 ceph-mon[30328]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:30.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1253219967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:39:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:30 smithi038 ceph-mon[36743]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:30 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1253219967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:39:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:30 smithi038 ceph-mon[26454]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1253219967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:39:30.764 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724570 for osd.5 2024-04-26T04:39:30.765 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:31 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2683407618' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:39:31.425 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:31 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2683407618' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:39:31.425 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:31 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2683407618' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:39:31.455 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574895 for osd.1 2024-04-26T04:39:31.456 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:32.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:32 smithi038 ceph-mon[36743]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:32.196 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:32 smithi038 ceph-mon[26454]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:32 smithi055 ceph-mon[30328]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:33.078 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411370 for osd.2 2024-04-26T04:39:33.078 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:33.188 INFO:teuthology.orchestra.run.smithi038.stdout:236223201298 2024-04-26T04:39:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:33 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1875007858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:39:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:33 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1875007858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:39:33.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:33 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1875007858' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:39:33.789 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201298 for osd.7 2024-04-26T04:39:33.789 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:34 smithi055 ceph-mon[30328]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:34 smithi038 ceph-mon[26454]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:34 smithi038 ceph-mon[36743]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:35.111 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:35.807 INFO:teuthology.orchestra.run.smithi038.stdout:111669149734 2024-04-26T04:39:35.829 INFO:teuthology.orchestra.run.smithi038.stdout:133143986209 2024-04-26T04:39:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:36 smithi038 ceph-mon[26454]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:36 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2220321530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:39:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:36 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/94128665' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:39:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:36 smithi038 ceph-mon[36743]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:36 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2220321530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:39:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:36 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/94128665' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:39:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:36 smithi055 ceph-mon[30328]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:36 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2220321530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:39:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:36 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/94128665' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:39:36.656 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149732 got 111669149734 for osd.3 2024-04-26T04:39:36.656 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:37.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:39:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:39:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:38.071 INFO:teuthology.orchestra.run.smithi038.stdout:201863462936 2024-04-26T04:39:38.417 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:38 smithi055 ceph-mon[30328]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:38.417 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/136662313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:39:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:38 smithi038 ceph-mon[26454]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/136662313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:39:38.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:38 smithi038 ceph-mon[36743]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:38.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/136662313' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:39:38.602 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462936 for osd.6 2024-04-26T04:39:38.603 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:39.274 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986206 got 133143986209 for osd.4 2024-04-26T04:39:39.274 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:39.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:40.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[26454]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:40.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:40.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:40.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[36743]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:40.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:40.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:40.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:40 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:40 smithi055 ceph-mon[30328]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:40 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:40 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:40 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:39:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:40 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:39:40.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:40 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:41.022 INFO:teuthology.orchestra.run.smithi038.stdout:34359738422 2024-04-26T04:39:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:41 smithi038 ceph-mon[36743]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:39:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:41 smithi038 ceph-mon[36743]: Reconfiguring daemon alertmanager.a on smithi038 2024-04-26T04:39:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:41 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2457887633' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:39:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:41 smithi038 ceph-mon[26454]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:39:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:41 smithi038 ceph-mon[26454]: Reconfiguring daemon alertmanager.a on smithi038 2024-04-26T04:39:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:41 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2457887633' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:39:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:41 smithi055 ceph-mon[30328]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:39:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:41 smithi055 ceph-mon[30328]: Reconfiguring daemon alertmanager.a on smithi038 2024-04-26T04:39:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:41 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2457887633' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:39:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:42 smithi038 ceph-mon[36743]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:42 smithi038 ceph-mon[26454]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:42 smithi055 ceph-mon[30328]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:43.224 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:42 smithi038 systemd[1]: Stopping Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:39:43.331 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738419 got 34359738422 for osd.0 2024-04-26T04:39:43.332 DEBUG:teuthology.parallel:result is None 2024-04-26T04:39:43.332 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:39:43.332 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:39:43.897 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[65998]: ts=2024-04-26T04:39:43.636Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:39:43.897 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:43 smithi038 podman[70160]: 2024-04-26 04:39:43.677445321 +0000 UTC m=+0.148319927 container died 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:44.034 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:44.159 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:43 smithi038 podman[70160]: 2024-04-26 04:39:43.978676829 +0000 UTC m=+0.449551446 container cleanup 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:44.159 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:43 smithi038 bash[70160]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:39:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:44 smithi038 ceph-mon[26454]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:44 smithi038 ceph-mon[36743]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:44.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:44 smithi038 podman[70209]: 2024-04-26 04:39:44.24664545 +0000 UTC m=+0.566039363 container remove 1463f2e78e0b182e9ed492eb1e08dad5607ce3d3c421cc301b15e3ffb7dad093 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:44.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:44 smithi038 podman[70209]: 2024-04-26 04:39:44.313985081 +0000 UTC m=+0.633378951 volume remove 1d6212a4ac4235d8ff16fbba37fd4f1a304faf4377e937489995858a7135cbb1 2024-04-26T04:39:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:44 smithi055 ceph-mon[30328]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:45.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:45 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:39:45.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:45 smithi038 systemd[1]: Stopped Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:39:45.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:45 smithi038 systemd[1]: Starting Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:39:46.189 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:39:46.190 INFO:teuthology.orchestra.run.smithi038.stderr:dumped all 2024-04-26T04:39:46.398 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:46 smithi038 ceph-mon[26454]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:46.398 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:46 smithi038 ceph-mon[36743]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:46.398 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:46 smithi038 podman[70390]: 2024-04-26 04:39:46.134049094 +0000 UTC m=+0.108564238 volume create 0c7f9c8766cc04844b6905a4b52accaf8a1a5f3d5738a33c83bc718a91f2f5e1 2024-04-26T04:39:46.398 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:46 smithi038 podman[70390]: 2024-04-26 04:39:46.044479523 +0000 UTC m=+0.018994684 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:39:46.398 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:46 smithi038 podman[70390]: 2024-04-26 04:39:46.20977308 +0000 UTC m=+0.184288222 container create 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:46 smithi055 ceph-mon[30328]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:47.000 INFO:teuthology.orchestra.run.smithi038.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-04-26T04:39:44.981443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385572,"kb_used_data":6708,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575644,"statfs":{"total":767960285184,"available":765517459456,"internally_reserved":0,"allocated":6868992,"data_stored":3897784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002482"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292645+0000","last_change":"2024-04-26T04:38:54.064213+0000","last_active":"2024-04-26T04:38:55.292645+0000","last_peered":"2024-04-26T04:38:55.292645+0000","last_clean":"2024-04-26T04:38:55.292645+0000","last_became_active":"2024-04-26T04:38:54.063797+0000","last_became_peered":"2024-04-26T04:38:54.063797+0000","last_unstale":"2024-04-26T04:38:55.292645+0000","last_undegraded":"2024-04-26T04:38:55.292645+0000","last_fullsized":"2024-04-26T04:38:55.292645+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:21.370670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292589+0000","last_change":"2024-04-26T04:38:54.530923+0000","last_active":"2024-04-26T04:38:55.292589+0000","last_peered":"2024-04-26T04:38:55.292589+0000","last_clean":"2024-04-26T04:38:55.292589+0000","last_became_active":"2024-04-26T04:38:54.530540+0000","last_became_peered":"2024-04-26T04:38:54.530540+0000","last_unstale":"2024-04-26T04:38:55.292589+0000","last_undegraded":"2024-04-26T04:38:55.292589+0000","last_fullsized":"2024-04-26T04:38:55.292589+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:28:56.675606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.069867+0000","last_change":"2024-04-26T04:38:54.530974+0000","last_active":"2024-04-26T04:38:55.069867+0000","last_peered":"2024-04-26T04:38:55.069867+0000","last_clean":"2024-04-26T04:38:55.069867+0000","last_became_active":"2024-04-26T04:38:54.530538+0000","last_became_peered":"2024-04-26T04:38:54.530538+0000","last_unstale":"2024-04-26T04:38:55.069867+0000","last_undegraded":"2024-04-26T04:38:55.069867+0000","last_fullsized":"2024-04-26T04:38:55.069867+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:40:04.608877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.069864+0000","last_change":"2024-04-26T04:38:54.065770+0000","last_active":"2024-04-26T04:38:55.069864+0000","last_peered":"2024-04-26T04:38:55.069864+0000","last_clean":"2024-04-26T04:38:55.069864+0000","last_became_active":"2024-04-26T04:38:54.065409+0000","last_became_peered":"2024-04-26T04:38:54.065409+0000","last_unstale":"2024-04-26T04:38:55.069864+0000","last_undegraded":"2024-04-26T04:38:55.069864+0000","last_fullsized":"2024-04-26T04:38:55.069864+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:56.230650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.069975+0000","last_change":"2024-04-26T04:38:11.717965+0000","last_active":"2024-04-26T04:38:55.069975+0000","last_peered":"2024-04-26T04:38:55.069975+0000","last_clean":"2024-04-26T04:38:55.069975+0000","last_became_active":"2024-04-26T04:38:11.717490+0000","last_became_peered":"2024-04-26T04:38:11.717490+0000","last_unstale":"2024-04-26T04:38:55.069975+0000","last_undegraded":"2024-04-26T04:38:55.069975+0000","last_fullsized":"2024-04-26T04:38:55.069975+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_clean_scrub_stamp":"2024-04-26T04:36:10.749042+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:18:38.216134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292501+0000","last_change":"2024-04-26T04:38:54.063499+0000","last_active":"2024-04-26T04:38:55.292501+0000","last_peered":"2024-04-26T04:38:55.292501+0000","last_clean":"2024-04-26T04:38:55.292501+0000","last_became_active":"2024-04-26T04:38:54.063035+0000","last_became_peered":"2024-04-26T04:38:54.063035+0000","last_unstale":"2024-04-26T04:38:55.292501+0000","last_undegraded":"2024-04-26T04:38:55.292501+0000","last_fullsized":"2024-04-26T04:38:55.292501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:20.279953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.067608+0000","last_change":"2024-04-26T04:38:54.066302+0000","last_active":"2024-04-26T04:38:55.067608+0000","last_peered":"2024-04-26T04:38:55.067608+0000","last_clean":"2024-04-26T04:38:55.067608+0000","last_became_active":"2024-04-26T04:38:54.065974+0000","last_became_peered":"2024-04-26T04:38:54.065974+0000","last_unstale":"2024-04-26T04:38:55.067608+0000","last_undegraded":"2024-04-26T04:38:55.067608+0000","last_fullsized":"2024-04-26T04:38:55.067608+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:27:22.809404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292498+0000","last_change":"2024-04-26T04:38:54.064959+0000","last_active":"2024-04-26T04:38:55.292498+0000","last_peered":"2024-04-26T04:38:55.292498+0000","last_clean":"2024-04-26T04:38:55.292498+0000","last_became_active":"2024-04-26T04:38:54.064530+0000","last_became_peered":"2024-04-26T04:38:54.064530+0000","last_unstale":"2024-04-26T04:38:55.292498+0000","last_undegraded":"2024-04-26T04:38:55.292498+0000","last_fullsized":"2024-04-26T04:38:55.292498+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:20.559351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292502+0000","last_change":"2024-04-26T04:38:54.065701+0000","last_active":"2024-04-26T04:38:55.292502+0000","last_peered":"2024-04-26T04:38:55.292502+0000","last_clean":"2024-04-26T04:38:55.292502+0000","last_became_active":"2024-04-26T04:38:54.065370+0000","last_became_peered":"2024-04-26T04:38:54.065370+0000","last_unstale":"2024-04-26T04:38:55.292502+0000","last_undegraded":"2024-04-26T04:38:55.292502+0000","last_fullsized":"2024-04-26T04:38:55.292502+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:38.001692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":855872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.59199999999999997}]},{"osd":1,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.66100000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.621},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.51600000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.70499999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.73399999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.68999999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.71899999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56299999999999994}]},{"osd":1,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.68899999999999995}]},{"osd":2,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.61699999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.64900000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.82199999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:39:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.72699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:38:47 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.35999999999999999,"15min":0.32900000000000001},"min":{"1min":0.29899999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.379,"15min":0.35699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.32000000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.42899999999999999,"15min":0.41899999999999998},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.27900000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:38:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.495,"15min":0.498},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.51000000000000001,"15min":0.51100000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.34999999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.63700000000000001,"15min":0.63300000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1,"5min":1,"15min":1},"last":0.61799999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:39:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.63,"15min":0.63500000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.91200000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.63300000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:38:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.68600000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.069,"5min":1.069,"15min":1.069},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.59599999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:38:50 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.35799999999999998,"15min":0.34599999999999997},"min":{"1min":0.25700000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.54500000000000004,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.40100000000000002,"15min":0.38100000000000001},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.36599999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:39:10 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.434,"15min":0.42299999999999999},"min":{"1min":0.30599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.43099999999999999,"15min":0.433},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.32200000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:38:38 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.377}]},{"osd":4,"last update":"Fri Apr 26 04:38:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.45200000000000001,"5min":0.437,"15min":0.437},"max":{"1min":0.81999999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.76000000000000001,"5min":0.872,"15min":0.872},"last":0.42999999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:39:21 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.62,"15min":0.621},"min":{"1min":0.46500000000000002,"5min":0.442,"15min":0.442},"max":{"1min":0.85299999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.61499999999999999,"15min":0.625},"min":{"1min":0.435,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72199999999999998,"5min":0.874,"15min":0.874},"last":0.53000000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:38:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.48299999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.46500000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.39000000000000001},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.42299999999999999,"15min":0.41299999999999998},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.33900000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.44600000000000001,"15min":0.439},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57299999999999995,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.45600000000000002,"15min":0.44800000000000001},"min":{"1min":0.29599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.67200000000000004,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.29899999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.48199999999999998,"15min":0.48499999999999999},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.626,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.441},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.501,"15min":0.503},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.59799999999999998,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.372}]},{"osd":4,"last update":"Fri Apr 26 04:39:03 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60399999999999998,"15min":0.60699999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.65200000000000002,"15min":0.65500000000000003},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.83599999999999997,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.60599999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.628,"15min":0.63700000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.504,"5min":0.58499999999999996,"15min":0.59899999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.69199999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.45000000000000001}]},{"osd":6,"last update":"Fri Apr 26 04:38:47 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.505}]},{"osd":7,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68799999999999994}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.41099999999999998,"15min":0.40999999999999998},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.433},{"interface":"front","average":{"1min":0.378,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56399999999999995,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.44500000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.41599999999999998,"15min":0.40999999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.497,"15min":0.51100000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.39500000000000002}]},{"osd":2,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.44800000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45900000000000002,"15min":0.46999999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.57999999999999996,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.372}]},{"osd":4,"last update":"Fri Apr 26 04:38:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59099999999999997,"15min":0.58499999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.45500000000000002,"5min":0.442,"15min":0.442},"max":{"1min":0.748,"5min":1.034,"15min":1.034},"last":0.51600000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79900000000000004,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.63400000000000001,"15min":0.64100000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89900000000000002,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.48799999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.499}]},{"osd":7,"last update":"Fri Apr 26 04:39:10 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.61799999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.1779999999999999,"15min":1.284},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.79800000000000004,"5min":15.547000000000001,"15min":15.547000000000001},"last":0.443},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.61099999999999999,"15min":0.623},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.73699999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.34000000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56699999999999995,"15min":0.57699999999999996},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.67600000000000005,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.76000000000000001,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.38900000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.80300000000000005,"5min":0.873,"15min":0.873},"last":0.496},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.60899999999999999,"15min":0.61699999999999999},"min":{"1min":0.36899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72599999999999998,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.50700000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.59099999999999997,"15min":0.59399999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.748,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.60099999999999998,"15min":0.60899999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.70899999999999996,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.46600000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.43099999999999999,"15min":0.433},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.621,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.497,"15min":0.49399999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.63,"5min":0.753,"15min":0.753},"last":0.72999999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63700000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:39:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.59299999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73599999999999999,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.83099999999999996,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.496}]},{"osd":1,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.57099999999999995,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69999999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.58899999999999997,"15min":0.59099999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78900000000000003,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.54100000000000004}]},{"osd":2,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.58899999999999997},"min":{"1min":0.40999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.41199999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76100000000000001,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.44600000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79200000000000004,"5min":0.874,"15min":0.874},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.58499999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.80700000000000005,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.48299999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.32900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.71499999999999997,"5min":0.745,"15min":0.745},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.45800000000000002,"15min":0.44500000000000001},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.378}]},{"osd":6,"last update":"Fri Apr 26 04:38:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.35799999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.173,"5min":1.173,"15min":1.173},"last":0.42599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:39:47.002 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:39:47.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:39:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:39:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:39:47.287 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:47.401 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-mon[26454]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:47.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-mon[36743]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 podman[70390]: 2024-04-26 04:39:47.100417087 +0000 UTC m=+1.074932249 container init 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 podman[70390]: 2024-04-26 04:39:47.105494543 +0000 UTC m=+1.080009692 container start 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.118Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.118Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.119Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.38 port=9094 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.120Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.153Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.154Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:39:47.402 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.157Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:39:47.403 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:47.157Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:39:47.403 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 bash[70390]: 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 2024-04-26T04:39:47.403 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:47 smithi038 systemd[1]: Started Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:39:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:47 smithi055 ceph-mon[30328]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:48 smithi038 ceph-mon[36743]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:48 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:48 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:48 smithi038 ceph-mon[26454]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:48 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:48 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:48.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:48 smithi055 ceph-mon[30328]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:48.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:48 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:48.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:48 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:49.294 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:39:49.296 INFO:teuthology.orchestra.run.smithi038.stderr:dumped all 2024-04-26T04:39:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:49 smithi055 ceph-mon[30328]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:39:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:49 smithi055 ceph-mon[30328]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:39:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:49 smithi038 ceph-mon[26454]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:39:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:49 smithi038 ceph-mon[26454]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:39:49.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:49.121Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000502158s 2024-04-26T04:39:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:49 smithi038 ceph-mon[36743]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:39:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:49 smithi038 ceph-mon[36743]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:39:49.910 INFO:teuthology.orchestra.run.smithi038.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-04-26T04:39:48.982200+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385572,"kb_used_data":6708,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575644,"statfs":{"total":767960285184,"available":765517459456,"internally_reserved":0,"allocated":6868992,"data_stored":3897784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002292"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292645+0000","last_change":"2024-04-26T04:38:54.064213+0000","last_active":"2024-04-26T04:38:55.292645+0000","last_peered":"2024-04-26T04:38:55.292645+0000","last_clean":"2024-04-26T04:38:55.292645+0000","last_became_active":"2024-04-26T04:38:54.063797+0000","last_became_peered":"2024-04-26T04:38:54.063797+0000","last_unstale":"2024-04-26T04:38:55.292645+0000","last_undegraded":"2024-04-26T04:38:55.292645+0000","last_fullsized":"2024-04-26T04:38:55.292645+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:21.370670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292589+0000","last_change":"2024-04-26T04:38:54.530923+0000","last_active":"2024-04-26T04:38:55.292589+0000","last_peered":"2024-04-26T04:38:55.292589+0000","last_clean":"2024-04-26T04:38:55.292589+0000","last_became_active":"2024-04-26T04:38:54.530540+0000","last_became_peered":"2024-04-26T04:38:54.530540+0000","last_unstale":"2024-04-26T04:38:55.292589+0000","last_undegraded":"2024-04-26T04:38:55.292589+0000","last_fullsized":"2024-04-26T04:38:55.292589+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:28:56.675606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.069867+0000","last_change":"2024-04-26T04:38:54.530974+0000","last_active":"2024-04-26T04:38:55.069867+0000","last_peered":"2024-04-26T04:38:55.069867+0000","last_clean":"2024-04-26T04:38:55.069867+0000","last_became_active":"2024-04-26T04:38:54.530538+0000","last_became_peered":"2024-04-26T04:38:54.530538+0000","last_unstale":"2024-04-26T04:38:55.069867+0000","last_undegraded":"2024-04-26T04:38:55.069867+0000","last_fullsized":"2024-04-26T04:38:55.069867+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:40:04.608877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.069864+0000","last_change":"2024-04-26T04:38:54.065770+0000","last_active":"2024-04-26T04:38:55.069864+0000","last_peered":"2024-04-26T04:38:55.069864+0000","last_clean":"2024-04-26T04:38:55.069864+0000","last_became_active":"2024-04-26T04:38:54.065409+0000","last_became_peered":"2024-04-26T04:38:54.065409+0000","last_unstale":"2024-04-26T04:38:55.069864+0000","last_undegraded":"2024-04-26T04:38:55.069864+0000","last_fullsized":"2024-04-26T04:38:55.069864+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:56.230650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.069975+0000","last_change":"2024-04-26T04:38:11.717965+0000","last_active":"2024-04-26T04:38:55.069975+0000","last_peered":"2024-04-26T04:38:55.069975+0000","last_clean":"2024-04-26T04:38:55.069975+0000","last_became_active":"2024-04-26T04:38:11.717490+0000","last_became_peered":"2024-04-26T04:38:11.717490+0000","last_unstale":"2024-04-26T04:38:55.069975+0000","last_undegraded":"2024-04-26T04:38:55.069975+0000","last_fullsized":"2024-04-26T04:38:55.069975+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_clean_scrub_stamp":"2024-04-26T04:36:10.749042+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:18:38.216134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292501+0000","last_change":"2024-04-26T04:38:54.063499+0000","last_active":"2024-04-26T04:38:55.292501+0000","last_peered":"2024-04-26T04:38:55.292501+0000","last_clean":"2024-04-26T04:38:55.292501+0000","last_became_active":"2024-04-26T04:38:54.063035+0000","last_became_peered":"2024-04-26T04:38:54.063035+0000","last_unstale":"2024-04-26T04:38:55.292501+0000","last_undegraded":"2024-04-26T04:38:55.292501+0000","last_fullsized":"2024-04-26T04:38:55.292501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:20.279953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.067608+0000","last_change":"2024-04-26T04:38:54.066302+0000","last_active":"2024-04-26T04:38:55.067608+0000","last_peered":"2024-04-26T04:38:55.067608+0000","last_clean":"2024-04-26T04:38:55.067608+0000","last_became_active":"2024-04-26T04:38:54.065974+0000","last_became_peered":"2024-04-26T04:38:54.065974+0000","last_unstale":"2024-04-26T04:38:55.067608+0000","last_undegraded":"2024-04-26T04:38:55.067608+0000","last_fullsized":"2024-04-26T04:38:55.067608+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:27:22.809404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292498+0000","last_change":"2024-04-26T04:38:54.064959+0000","last_active":"2024-04-26T04:38:55.292498+0000","last_peered":"2024-04-26T04:38:55.292498+0000","last_clean":"2024-04-26T04:38:55.292498+0000","last_became_active":"2024-04-26T04:38:54.064530+0000","last_became_peered":"2024-04-26T04:38:54.064530+0000","last_unstale":"2024-04-26T04:38:55.292498+0000","last_undegraded":"2024-04-26T04:38:55.292498+0000","last_fullsized":"2024-04-26T04:38:55.292498+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:20.559351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-26T04:38:55.292502+0000","last_change":"2024-04-26T04:38:54.065701+0000","last_active":"2024-04-26T04:38:55.292502+0000","last_peered":"2024-04-26T04:38:55.292502+0000","last_clean":"2024-04-26T04:38:55.292502+0000","last_became_active":"2024-04-26T04:38:54.065370+0000","last_became_peered":"2024-04-26T04:38:54.065370+0000","last_unstale":"2024-04-26T04:38:55.292502+0000","last_undegraded":"2024-04-26T04:38:55.292502+0000","last_fullsized":"2024-04-26T04:38:55.292502+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:38.001692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298552,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446600,"statfs":{"total":95995035648,"available":95689318400,"internally_reserved":0,"allocated":1224704,"data_stored":855872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.504},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.60399999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.49099999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.625},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.46999999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.53700000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.378},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.40699999999999997}]},{"osd":5,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.58599999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.66900000000000004}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.54500000000000004}]},{"osd":1,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.60599999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.58999999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.56200000000000006}]},{"osd":4,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.51100000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63}]},{"osd":7,"last update":"Fri Apr 26 04:39:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.621}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:38:47 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.35999999999999999,"15min":0.32900000000000001},"min":{"1min":0.29899999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.379,"15min":0.35699999999999998},"min":{"1min":0.28100000000000003,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.373}]},{"osd":2,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.42899999999999999,"15min":0.41899999999999998},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"min":{"1min":0.29399999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.47199999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:39:40 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.48299999999999998,"15min":0.49399999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72799999999999998,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.49399999999999999,"15min":0.50600000000000001},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69999999999999996,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.41299999999999998}]},{"osd":4,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.63700000000000001,"15min":0.63300000000000001},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.151,"5min":1.151,"15min":1.151},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1,"5min":1,"15min":1},"last":0.57099999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:39:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.61599999999999999,"15min":0.61399999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.63,"15min":0.63500000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.91200000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.59499999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:38:47 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.502},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.55500000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:39:09 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.069,"5min":1.069,"15min":1.069},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.624}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:38:50 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.35799999999999998,"15min":0.34599999999999997},"min":{"1min":0.25700000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.54500000000000004,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.40100000000000002,"15min":0.38100000000000001},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.47099999999999997}]},{"osd":2,"last update":"Fri Apr 26 04:39:10 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.434,"15min":0.42299999999999999},"min":{"1min":0.30599999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.43099999999999999,"15min":0.433},"min":{"1min":0.30499999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.26200000000000001}]},{"osd":3,"last update":"Fri Apr 26 04:39:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.5,"15min":0.50600000000000001},"min":{"1min":0.35599999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.61499999999999999,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.46999999999999997,"15min":0.47499999999999998},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.67900000000000005,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.37}]},{"osd":4,"last update":"Fri Apr 26 04:38:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.45200000000000001,"5min":0.437,"15min":0.437},"max":{"1min":0.81999999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.76000000000000001,"5min":0.872,"15min":0.872},"last":0.44800000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:39:21 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.62,"15min":0.621},"min":{"1min":0.46500000000000002,"5min":0.442,"15min":0.442},"max":{"1min":0.85299999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.61499999999999999,"15min":0.625},"min":{"1min":0.435,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72199999999999998,"5min":0.874,"15min":0.874},"last":0.48599999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:38:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.438},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.65000000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:39:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.495},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.70199999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.39000000000000001},"min":{"1min":0.27300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.25},{"interface":"front","average":{"1min":0.433,"5min":0.42299999999999999,"15min":0.41299999999999998},"min":{"1min":0.27300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.35699999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.44600000000000001,"15min":0.439},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.57299999999999995,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.45600000000000002,"15min":0.44800000000000001},"min":{"1min":0.29599999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.67200000000000004,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.314}]},{"osd":3,"last update":"Fri Apr 26 04:38:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.48199999999999998,"15min":0.48499999999999999},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.626,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.501,"15min":0.503},"min":{"1min":0.35399999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.59799999999999998,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.379}]},{"osd":4,"last update":"Fri Apr 26 04:39:03 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60399999999999998,"15min":0.60699999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.65200000000000002,"15min":0.65500000000000003},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.83599999999999997,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.85199999999999998}]},{"osd":5,"last update":"Fri Apr 26 04:39:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.628,"15min":0.63700000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.504,"5min":0.58499999999999996,"15min":0.59899999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.69199999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.77500000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:38:47 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.67400000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.75},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.58699999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.41099999999999998,"15min":0.40999999999999998},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.1799999999999999,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.378,"5min":0.40100000000000002,"15min":0.39900000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56399999999999995,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.31900000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.41599999999999998,"15min":0.40999999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.497,"15min":0.51100000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.38800000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:39:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.44800000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45900000000000002,"15min":0.46999999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.57999999999999996,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.27900000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:38:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.59099999999999997,"15min":0.58499999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.65000000000000002,"15min":0.65800000000000003},"min":{"1min":0.45500000000000002,"5min":0.442,"15min":0.442},"max":{"1min":0.748,"5min":1.034,"15min":1.034},"last":0.70799999999999996}]},{"osd":5,"last update":"Fri Apr 26 04:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.64600000000000002,"15min":0.65700000000000003},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79900000000000004,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.63400000000000001,"15min":0.64100000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89900000000000002,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.71999999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.56599999999999995}]},{"osd":7,"last update":"Fri Apr 26 04:39:10 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.014,"5min":1.014,"15min":1.014},"last":0.68100000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.1779999999999999,"15min":1.284},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.79800000000000004,"5min":15.547000000000001,"15min":15.547000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.61099999999999999,"15min":0.623},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.73699999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.35899999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56699999999999995,"15min":0.57699999999999996},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.67600000000000005,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.59199999999999997,"15min":0.59799999999999998},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.76000000000000001,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.47899999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59099999999999997,"15min":0.59699999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.80300000000000005,"5min":0.873,"15min":0.873},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.60899999999999999,"15min":0.61699999999999999},"min":{"1min":0.36899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72599999999999998,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.45800000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:38:59 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.59099999999999997,"15min":0.59399999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.748,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.5},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.60099999999999998,"15min":0.60899999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.70899999999999996,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.44500000000000001}]},{"osd":5,"last update":"Fri Apr 26 04:39:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.43099999999999999,"15min":0.433},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.621,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.497,"15min":0.49399999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.63,"5min":0.753,"15min":0.753},"last":0.56299999999999994}]},{"osd":6,"last update":"Fri Apr 26 04:38:46 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.64100000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:39:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.57999999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36099999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73599999999999999,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.57599999999999996,"15min":0.58099999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.83099999999999996,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.60199999999999998}]},{"osd":1,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.57099999999999995,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.69999999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.58899999999999997,"15min":0.59099999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78900000000000003,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.55600000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.59199999999999997,"15min":0.58899999999999997},"min":{"1min":0.40999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57099999999999995,"15min":0.57199999999999995},"min":{"1min":0.41199999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76100000000000001,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.72599999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79200000000000004,"5min":0.874,"15min":0.874},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.58499999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.80700000000000005,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.626}]},{"osd":4,"last update":"Fri Apr 26 04:39:24 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.32900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.71499999999999997,"5min":0.745,"15min":0.745},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.45800000000000002,"15min":0.44500000000000001},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.244,"5min":1.244,"15min":1.244},"last":0.41799999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:38:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.437}]},{"osd":7,"last update":"Fri Apr 26 04:39:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.173,"5min":1.173,"15min":1.173},"last":0.45700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:39:49.911 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:39:49.911 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:39:49.912 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:39:49.912 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:39:50.177 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:50.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:50 smithi055 ceph-mon[30328]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:50 smithi038 ceph-mon[26454]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:50.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:50 smithi038 ceph-mon[36743]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:51.323 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:51 smithi038 ceph-mon[26454]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:51.323 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:51 smithi038 ceph-mon[26454]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:51.324 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:51 smithi038 ceph-mon[36743]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:51.324 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:51 smithi038 ceph-mon[36743]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:51.587 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:51 smithi055 ceph-mon[30328]: from='client.14613 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:39:51.587 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:51 smithi055 ceph-mon[30328]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:52.042 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:39:52.042 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:39:52.314 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:52 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2648223203' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:39:52.315 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:52 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2648223203' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:39:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:52 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2648223203' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:39:52.711 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:39:52.711 INFO:tasks.cephadm:Setup complete, yielding 2024-04-26T04:39:52.711 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-26T04:39:52.722 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi038.front.sepia.ceph.com 2024-04-26T04:39:52.723 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-26T04:39:52.991 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-mon[30328]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:53.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 systemd[1]: Stopping Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:39:53.384 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:53 smithi038 ceph-mon[26454]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:53.385 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:53 smithi038 ceph-mon[36743]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:39:53.726 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.449Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:39:53.727 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.450Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:39:53.727 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.450Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:39:53.727 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.450Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:39:53.727 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[58230]: ts=2024-04-26T04:39:53.450Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:39:53.727 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 podman[61484]: 2024-04-26 04:39:53.459863657 +0000 UTC m=+0.130352843 container died 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:54.101 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 podman[61484]: 2024-04-26 04:39:53.832220996 +0000 UTC m=+0.502710191 container cleanup 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:54.101 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:53 smithi055 bash[61484]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:39:54.345 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:54 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:54.345 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:54 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:39:54.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 podman[61498]: 2024-04-26 04:39:54.100259396 +0000 UTC m=+0.642833196 container remove 3d462204d5514eaf466d65cd971ed4b7a61c29cf5b4494994616662d253f72ec (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:54.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:39:54.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 systemd[1]: Stopped Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:39:54.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.093s CPU time. 2024-04-26T04:39:54.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 systemd[1]: Starting Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 podman[61590]: 2024-04-26 04:39:54.48496213 +0000 UTC m=+0.108295417 container create f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 podman[61590]: 2024-04-26 04:39:54.396228081 +0000 UTC m=+0.019561371 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 podman[61590]: 2024-04-26 04:39:54.687171462 +0000 UTC m=+0.310504749 container init f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 podman[61590]: 2024-04-26 04:39:54.689856695 +0000 UTC m=+0.313189983 container start f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.714Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.714Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.715Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi055 (none))" 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.715Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.715Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.716Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:39:54.738 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.717Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.718Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.718Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.720Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.720Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.471µs 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.720Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.720Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.720Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.720Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=55.329µs wal_replay_duration=646.153µs wbl_replay_duration=101ns total_replay_duration=742.645µs 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.721Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.721Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.721Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.736Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.283948ms db_storage=729ns remote_storage=1.063µs web_handler=317ns query_engine=536ns scrape=388.953µs scrape_sd=27.896µs notify=18.715µs notify_sd=6.177µs rules=14.629397ms tracing=5.114µs 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.736Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:39:54.739 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:39:54.736Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:39:55.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 bash[61590]: f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 2024-04-26T04:39:55.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:39:54 smithi055 systemd[1]: Started Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:39:55.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:55 smithi055 ceph-mon[30328]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:55.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:55 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:39:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:55 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:39:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:55 smithi055 ceph-mon[30328]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:55 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:55 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[26454]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[26454]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:55.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STOPPING 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STOPPED 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STARTING 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Serving on http://:::9283 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STARTED 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[36743]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[36743]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:55.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STOPPING 2024-04-26T04:39:56.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STOPPED 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STARTING 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Serving on http://:::9283 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STARTED 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STOPPING 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STOPPED 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STARTING 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Serving on http://:::9283 2024-04-26T04:39:56.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:39:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:39:55] ENGINE Bus STARTED 2024-04-26T04:39:56.566 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038:9093"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038:9093"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.567 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:39:56.568 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:39:56.568 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:39:56.568 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:39:56.568 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.568 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:56 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038:9093"}]: dispatch 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038:9093"}]: dispatch 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:39:56.714 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038:9093"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038:9093"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: osdmap e65: 8 total, 8 up, 8 in 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:39:56.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055.front.sepia.ceph.com:3000"}]: dispatch 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:39:56.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:39:56.304+0000 7fc828bf8700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:39:57.109 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-26T04:39:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:39:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: fsmap foo:0 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: Saving service mds.foo spec with placement count:2 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:57.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:57 smithi055 ceph-mon[30328]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:57.435 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:39:57.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:39:57.123Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002718696s 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:39:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: fsmap foo:0 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: Saving service mds.foo spec with placement count:2 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[36743]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: osdmap e66: 8 total, 8 up, 8 in 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: osdmap e67: 8 total, 8 up, 8 in 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-26T04:39:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: fsmap foo:0 2024-04-26T04:39:57.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: Saving service mds.foo spec with placement count:2 2024-04-26T04:39:57.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:39:57.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:57 smithi038 ceph-mon[26454]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:39:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:39:58 smithi038 ceph-mon[26454]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:39:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:39:58 smithi038 ceph-mon[36743]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:39:58.591 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:39:58 smithi055 ceph-mon[30328]: osdmap e68: 8 total, 8 up, 8 in 2024-04-26T04:40:00.043 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:39:59.998+0000 7fc82b3fd700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1 filesystem is offline; 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.043 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:39:59.998+0000 7fc82b3fd700 -1 log_channel(cluster) log [ERR] : [ERR] MDS_ALL_DOWN: 1 filesystem is offline 2024-04-26T04:40:00.043 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:39:59.998+0000 7fc82b3fd700 -1 log_channel(cluster) log [ERR] : fs foo is offline because no MDS is active for it. 2024-04-26T04:40:00.043 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:39:59.998+0000 7fc82b3fd700 -1 log_channel(cluster) log [ERR] : [WRN] MDS_UP_LESS_THAN_MAX: 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.043 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:39:59.998+0000 7fc82b3fd700 -1 log_channel(cluster) log [ERR] : fs foo has 0 MDS online, but wants 1 2024-04-26T04:40:00.071 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-26T04:40:00.424 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: pgmap v51: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: Health detail: HEALTH_ERR 1 filesystem is offline; 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: [ERR] MDS_ALL_DOWN: 1 filesystem is offline 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: fs foo is offline because no MDS is active for it. 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: [WRN] MDS_UP_LESS_THAN_MAX: 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[26454]: fs foo has 0 MDS online, but wants 1 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: pgmap v51: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: Health detail: HEALTH_ERR 1 filesystem is offline; 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: [ERR] MDS_ALL_DOWN: 1 filesystem is offline 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: fs foo is offline because no MDS is active for it. 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: [WRN] MDS_UP_LESS_THAN_MAX: 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:00 smithi038 ceph-mon[36743]: fs foo has 0 MDS online, but wants 1 2024-04-26T04:40:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: pgmap v51: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: Health detail: HEALTH_ERR 1 filesystem is offline; 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: [ERR] MDS_ALL_DOWN: 1 filesystem is offline 2024-04-26T04:40:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: fs foo is offline because no MDS is active for it. 2024-04-26T04:40:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: [WRN] MDS_UP_LESS_THAN_MAX: 1 filesystem is online with fewer MDS than max_mds 2024-04-26T04:40:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:00 smithi055 ceph-mon[30328]: fs foo has 0 MDS online, but wants 1 2024-04-26T04:40:02.287 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:02.287 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.288 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:02.289 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:02 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:02.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:02 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:03 smithi038 ceph-mon[26454]: Deploying daemon mds.foo.smithi038.qsffiu on smithi038 2024-04-26T04:40:03.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:03 smithi038 ceph-mon[36743]: Deploying daemon mds.foo.smithi038.qsffiu on smithi038 2024-04-26T04:40:03.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:03 smithi055 ceph-mon[30328]: Deploying daemon mds.foo.smithi038.qsffiu on smithi038 2024-04-26T04:40:03.673 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:40:03.683 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:40:03.683 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:40:03.683 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:40:04.202 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:04.309 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:04 smithi038 ceph-mon[26454]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:04.309 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:03] "GET /metrics HTTP/1.1" 200 203238 "" "Prometheus/2.43.0" 2024-04-26T04:40:04.310 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:04 smithi038 ceph-mon[36743]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:04 smithi055 ceph-mon[30328]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:06.050 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:40:06.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:06 smithi038 ceph-mon[26454]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:06.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:06 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2554723819' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:40:06.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:06 smithi038 ceph-mon[36743]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:06.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:06 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2554723819' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:40:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:06 smithi055 ceph-mon[30328]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:06 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2554723819' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T04:40:06.839 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":20,"flags":0,"active_gid":24332,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6800","nonce":3488049706},{"type":"v1","addr":"172.21.15.38:6801","nonce":3488049706}]},"active_addr":"172.21.15.38:6801/3488049706","active_change":"2024-04-26T04:38:38.949817+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24358,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.38:8443/","prometheus":"http://172.21.15.38:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":3149555040}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":1564621640}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":4062968598}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":1699336096}]}]} 2024-04-26T04:40:06.844 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T04:40:06.844 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T04:40:06.844 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:40:06.894 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:40:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:07.115 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:07.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: Deploying daemon mds.foo.smithi055.cuaegn on smithi055 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:boot 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: daemon mds.foo.smithi038.qsffiu assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:40:07.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: Cluster is now healthy 2024-04-26T04:40:07.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:40:07.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: fsmap foo:0 1 up:standby 2024-04-26T04:40:07.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:creating} 2024-04-26T04:40:07.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[36743]: daemon mds.foo.smithi038.qsffiu is now active in filesystem foo as rank 0 2024-04-26T04:40:07.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.478 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.478 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:07.478 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:07.478 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: Deploying daemon mds.foo.smithi055.cuaegn on smithi055 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:boot 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: daemon mds.foo.smithi038.qsffiu assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:40:07.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:40:07.480 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: Cluster is now healthy 2024-04-26T04:40:07.480 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:40:07.480 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: fsmap foo:0 1 up:standby 2024-04-26T04:40:07.480 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:creating} 2024-04-26T04:40:07.481 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:07 smithi038 ceph-mon[26454]: daemon mds.foo.smithi038.qsffiu is now active in filesystem foo as rank 0 2024-04-26T04:40:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: Deploying daemon mds.foo.smithi055.cuaegn on smithi055 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:boot 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: daemon mds.foo.smithi038.qsffiu assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-26T04:40:07.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: Cluster is now healthy 2024-04-26T04:40:07.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:40:07.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: fsmap foo:0 1 up:standby 2024-04-26T04:40:07.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:creating} 2024-04-26T04:40:07.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:07 smithi055 ceph-mon[30328]: daemon mds.foo.smithi038.qsffiu is now active in filesystem foo as rank 0 2024-04-26T04:40:08.750 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:40:08.750 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":68,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:39:57.306486+0000","last_up_change":"2024-04-26T04:38:07.872498+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:38:52.695557+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:39:55.221857+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:39:55.718635+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6803","nonce":3619308679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6805","nonce":3619308679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6809","nonce":3619308679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6807","nonce":3619308679}]},"public_addr":"172.21.15.38:6803/3619308679","cluster_addr":"172.21.15.38:6805/3619308679","heartbeat_back_addr":"172.21.15.38:6809/3619308679","heartbeat_front_addr":"172.21.15.38:6807/3619308679","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6811","nonce":2181836257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6813","nonce":2181836257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6817","nonce":2181836257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6815","nonce":2181836257}]},"public_addr":"172.21.15.38:6811/2181836257","cluster_addr":"172.21.15.38:6813/2181836257","heartbeat_back_addr":"172.21.15.38:6817/2181836257","heartbeat_front_addr":"172.21.15.38:6815/2181836257","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6819","nonce":124975504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6821","nonce":124975504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6825","nonce":124975504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6823","nonce":124975504}]},"public_addr":"172.21.15.38:6819/124975504","cluster_addr":"172.21.15.38:6821/124975504","heartbeat_back_addr":"172.21.15.38:6825/124975504","heartbeat_front_addr":"172.21.15.38:6823/124975504","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6827","nonce":1698863327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6829","nonce":1698863327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6833","nonce":1698863327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6831","nonce":1698863327}]},"public_addr":"172.21.15.38:6827/1698863327","cluster_addr":"172.21.15.38:6829/1698863327","heartbeat_back_addr":"172.21.15.38:6833/1698863327","heartbeat_front_addr":"172.21.15.38:6831/1698863327","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:40:09.112 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:active 2024-04-26T04:40:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:active 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/306445459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/306445459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:09.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.508 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T04:40:09.508 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T04:40:09.509 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:active 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/306445459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:09.510 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:10.112 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[26454]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:10.356 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[26454]: mds.? [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] up:boot 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[26454]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[26454]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[36743]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[36743]: mds.? [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] up:boot 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:40:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:10 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:10.495 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:10 smithi055 ceph-mon[30328]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:10.495 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:10 smithi055 ceph-mon[30328]: mds.? [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] up:boot 2024-04-26T04:40:10.495 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:10 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:10.495 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:10 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:40:10.495 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:10 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:12.137 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:40:12.138 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":68,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:39:57.306486+0000","last_up_change":"2024-04-26T04:38:07.872498+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:38:52.695557+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:39:55.221857+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:39:55.718635+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6803","nonce":3619308679}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6805","nonce":3619308679}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6809","nonce":3619308679}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3619308679},{"type":"v1","addr":"172.21.15.38:6807","nonce":3619308679}]},"public_addr":"172.21.15.38:6803/3619308679","cluster_addr":"172.21.15.38:6805/3619308679","heartbeat_back_addr":"172.21.15.38:6809/3619308679","heartbeat_front_addr":"172.21.15.38:6807/3619308679","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6811","nonce":2181836257}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6813","nonce":2181836257}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6817","nonce":2181836257}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2181836257},{"type":"v1","addr":"172.21.15.38:6815","nonce":2181836257}]},"public_addr":"172.21.15.38:6811/2181836257","cluster_addr":"172.21.15.38:6813/2181836257","heartbeat_back_addr":"172.21.15.38:6817/2181836257","heartbeat_front_addr":"172.21.15.38:6815/2181836257","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6819","nonce":124975504}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6821","nonce":124975504}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6825","nonce":124975504}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":124975504},{"type":"v1","addr":"172.21.15.38:6823","nonce":124975504}]},"public_addr":"172.21.15.38:6819/124975504","cluster_addr":"172.21.15.38:6821/124975504","heartbeat_back_addr":"172.21.15.38:6825/124975504","heartbeat_front_addr":"172.21.15.38:6823/124975504","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6827","nonce":1698863327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6829","nonce":1698863327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6833","nonce":1698863327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1698863327},{"type":"v1","addr":"172.21.15.38:6831","nonce":1698863327}]},"public_addr":"172.21.15.38:6827/1698863327","cluster_addr":"172.21.15.38:6829/1698863327","heartbeat_back_addr":"172.21.15.38:6833/1698863327","heartbeat_front_addr":"172.21.15.38:6831/1698863327","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T04:40:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:12 smithi055 ceph-mon[30328]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:12.382 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:12 smithi038 ceph-mon[26454]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:12.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:12 smithi038 ceph-mon[36743]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:12.926 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T04:40:12.927 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T04:40:12.928 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T04:40:12.928 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T04:40:12.929 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T04:40:12.929 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T04:40:12.930 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T04:40:12.930 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T04:40:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[26454]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:active 2024-04-26T04:40:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[26454]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1048713306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:40:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[36743]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:active 2024-04-26T04:40:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1048713306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:40:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:13.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:13 smithi055 ceph-mon[30328]: mds.? [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] up:active 2024-04-26T04:40:13.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:13 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:13.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:13 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1048713306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T04:40:13.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:13 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:13.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:13 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:13.778 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:14.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:14 smithi038 ceph-mon[26454]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:14.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:14 smithi038 ceph-mon[26454]: mds.? [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] up:standby 2024-04-26T04:40:14.247 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:14 smithi038 ceph-mon[26454]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:14.248 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:13] "GET /metrics HTTP/1.1" 200 238524 "" "Prometheus/2.43.0" 2024-04-26T04:40:14.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:14 smithi038 ceph-mon[36743]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:14.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:14 smithi038 ceph-mon[36743]: mds.? [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] up:standby 2024-04-26T04:40:14.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:14 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:14.249 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:14.250 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:14.250 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:14.251 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:14 smithi055 ceph-mon[30328]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:14 smithi055 ceph-mon[30328]: mds.? [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] up:standby 2024-04-26T04:40:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:14 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:40:14.626 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:15.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:15 smithi055 ceph-mon[30328]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:15.661 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[36743]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:40:15.662 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:15 smithi038 ceph-mon[26454]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:16.945 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:40:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:17.015 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:18 smithi055 ceph-mon[30328]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:18 smithi038 ceph-mon[26454]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:18 smithi038 ceph-mon[36743]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:19.341 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:19 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:19.342 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:19 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:19.342 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:19 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:19.342 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:19 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:19 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:19 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:20.409 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:20 smithi038 ceph-mon[26454]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:20.409 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:20 smithi038 ceph-mon[36743]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:20 smithi055 ceph-mon[30328]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:21.353 INFO:teuthology.orchestra.run.smithi038.stdout:34359738431 2024-04-26T04:40:21.354 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T04:40:22.212 INFO:teuthology.orchestra.run.smithi038.stdout:133143986220 2024-04-26T04:40:22.213 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T04:40:22.451 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:22 smithi038 ceph-mon[36743]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:22.451 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:22 smithi038 ceph-mon[26454]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:22 smithi055 ceph-mon[30328]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-26T04:40:23.531 INFO:teuthology.orchestra.run.smithi038.stdout:77309411382 2024-04-26T04:40:23.532 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T04:40:24.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:24 smithi038 ceph-mon[26454]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:24.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:24 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:24.333 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:23 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:23] "GET /metrics HTTP/1.1" 200 238524 "" "Prometheus/2.43.0" 2024-04-26T04:40:24.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:24 smithi038 ceph-mon[36743]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:24.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:24 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:24.358 INFO:teuthology.orchestra.run.smithi038.stdout:55834574907 2024-04-26T04:40:24.358 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T04:40:24.579 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:24 smithi055 ceph-mon[30328]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:24 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:25.527 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:26 smithi038 ceph-mon[26454]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:26 smithi038 ceph-mon[36743]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:26 smithi055 ceph-mon[30328]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:27.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:40:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:27.746 INFO:teuthology.orchestra.run.smithi038.stdout:111669149746 2024-04-26T04:40:27.746 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T04:40:28.351 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:28 smithi038 ceph-mon[26454]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:28.351 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:28 smithi038 ceph-mon[36743]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:28 smithi055 ceph-mon[30328]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-26T04:40:29.121 INFO:teuthology.orchestra.run.smithi038.stdout:201863462947 2024-04-26T04:40:29.122 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T04:40:30.196 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:30.197 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:30.197 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:30.237 INFO:teuthology.orchestra.run.smithi038.stdout:236223201310 2024-04-26T04:40:30.238 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T04:40:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:30 smithi038 ceph-mon[26454]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:30 smithi038 ceph-mon[36743]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:30 smithi055 ceph-mon[30328]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:30.966 INFO:teuthology.orchestra.run.smithi038.stdout:167503724583 2024-04-26T04:40:30.967 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T04:40:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:32 smithi038 ceph-mon[26454]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:32 smithi038 ceph-mon[36743]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:32 smithi055 ceph-mon[30328]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:33.377 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:33 smithi038 ceph-mon[26454]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:33.377 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:33 smithi038 ceph-mon[36743]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:33.473 INFO:teuthology.orchestra.run.smithi038.stdout:34359738434 2024-04-26T04:40:33.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:33 smithi055 ceph-mon[30328]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:34.248 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:33] "GET /metrics HTTP/1.1" 200 239064 "" "Prometheus/2.43.0" 2024-04-26T04:40:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:34 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4155674090' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:40:34.671 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:34 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4155674090' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:40:34.672 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:34 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4155674090' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T04:40:34.699 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738431 got 34359738434 for osd.0 2024-04-26T04:40:34.699 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:35.337 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:35.337 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:35.419 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:35 smithi038 ceph-mon[26454]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:35.419 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:35 smithi038 ceph-mon[36743]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:35.521 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:35 smithi055 ceph-mon[30328]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:35.759 INFO:teuthology.orchestra.run.smithi038.stdout:77309411384 2024-04-26T04:40:36.395 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:36 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2940328433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:40:36.396 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:36 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2940328433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:40:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:36 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2940328433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T04:40:36.704 INFO:teuthology.orchestra.run.smithi038.stdout:55834574909 2024-04-26T04:40:36.976 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:40:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:37.536 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:37 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2382666992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:40:37.536 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:37 smithi038 ceph-mon[36743]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:37.537 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:37 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2382666992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:40:37.537 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:37 smithi038 ceph-mon[26454]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:37 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2382666992' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T04:40:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:37 smithi055 ceph-mon[30328]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:39.140 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574909 for osd.1 2024-04-26T04:40:39.140 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.355 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:40:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-26T04:40:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-26T04:40:39.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:39.777 INFO:teuthology.orchestra.run.smithi038.stdout:167503724585 2024-04-26T04:40:40.221 INFO:teuthology.orchestra.run.smithi038.stdout:133143986223 2024-04-26T04:40:40.221 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[26454]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:40.221 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-26T04:40:40.221 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-26T04:40:40.221 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T04:40:40.221 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[26454]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/678216543' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[36743]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[36743]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:40:40.222 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:40 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/678216543' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:40:40.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:40 smithi055 ceph-mon[30328]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:40.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:40 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-26T04:40:40.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:40 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-26T04:40:40.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:40 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-26T04:40:40.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:40 smithi055 ceph-mon[30328]: osdmap e69: 8 total, 8 up, 8 in 2024-04-26T04:40:40.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:40 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/678216543' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T04:40:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:41 smithi055 ceph-mon[30328]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:40:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:41 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1254477937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:40:41.408 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:41 smithi038 ceph-mon[26454]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:40:41.408 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:41 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1254477937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:40:41.408 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:41 smithi038 ceph-mon[36743]: osdmap e70: 8 total, 8 up, 8 in 2024-04-26T04:40:41.409 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:41 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1254477937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T04:40:41.433 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411382 got 77309411384 for osd.2 2024-04-26T04:40:41.433 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:41.995 INFO:teuthology.orchestra.run.smithi038.stdout:201863462950 2024-04-26T04:40:42.104 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986220 got 133143986223 for osd.4 2024-04-26T04:40:42.104 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:42 smithi055 ceph-mon[30328]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:42 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2007787042' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:40:42.391 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:42 smithi038 ceph-mon[26454]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:42.391 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:42 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2007787042' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:40:42.391 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:42 smithi038 ceph-mon[36743]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:42.391 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:42 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2007787042' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T04:40:42.727 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462947 got 201863462950 for osd.6 2024-04-26T04:40:42.727 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:42.792 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:43.446 INFO:teuthology.orchestra.run.smithi038.stdout:111669149749 2024-04-26T04:40:43.657 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724583 got 167503724585 for osd.5 2024-04-26T04:40:43.658 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:44.210 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:44 smithi038 ceph-mon[26454]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:44.210 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:44 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/519732755' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:40:44.211 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:43] "GET /metrics HTTP/1.1" 200 239041 "" "Prometheus/2.43.0" 2024-04-26T04:40:44.211 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:44 smithi038 ceph-mon[36743]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:44.211 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:44 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/519732755' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:40:44.212 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149749 for osd.3 2024-04-26T04:40:44.212 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:44 smithi055 ceph-mon[30328]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:44 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/519732755' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T04:40:46.270 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:46 smithi038 ceph-mon[36743]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:46.270 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:46 smithi038 ceph-mon[26454]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:46 smithi055 ceph-mon[30328]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:47.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:40:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:47.385 INFO:teuthology.orchestra.run.smithi038.stdout:236223201313 2024-04-26T04:40:48.096 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201310 got 236223201313 for osd.7 2024-04-26T04:40:48.096 DEBUG:teuthology.parallel:result is None 2024-04-26T04:40:48.096 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T04:40:48.097 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:40:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:48.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:48.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:48.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:47 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2021990503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:40:48.364 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2021990503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:40:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:40:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:40:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:40:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:48.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:47 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2021990503' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T04:40:50.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:50 smithi038 ceph-mon[26454]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:50.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:50 smithi038 ceph-mon[36743]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:50.280 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:40:50.282 INFO:teuthology.orchestra.run.smithi038.stderr:dumped all 2024-04-26T04:40:50.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:50 smithi055 ceph-mon[30328]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:50.798 INFO:teuthology.orchestra.run.smithi038.stdout:{"pg_ready":true,"pg_map":{"version":78,"stamp":"2024-04-26T04:40:48.994447+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386248,"kb_used_data":7380,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574968,"statfs":{"total":767960285184,"available":765516767232,"internally_reserved":0,"allocated":7557120,"data_stored":4252918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.940953"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286930+0000","last_change":"2024-04-26T04:39:56.321580+0000","last_active":"2024-04-26T04:40:39.286930+0000","last_peered":"2024-04-26T04:40:39.286930+0000","last_clean":"2024-04-26T04:40:39.286930+0000","last_became_active":"2024-04-26T04:39:56.321278+0000","last_became_peered":"2024-04-26T04:39:56.321278+0000","last_unstale":"2024-04-26T04:40:39.286930+0000","last_undegraded":"2024-04-26T04:40:39.286930+0000","last_fullsized":"2024-04-26T04:40:39.286930+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:43:21.733155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286763+0000","last_change":"2024-04-26T04:39:56.293313+0000","last_active":"2024-04-26T04:40:39.286763+0000","last_peered":"2024-04-26T04:40:39.286763+0000","last_clean":"2024-04-26T04:40:39.286763+0000","last_became_active":"2024-04-26T04:39:56.292965+0000","last_became_peered":"2024-04-26T04:39:56.292965+0000","last_unstale":"2024-04-26T04:40:39.286763+0000","last_undegraded":"2024-04-26T04:40:39.286763+0000","last_fullsized":"2024-04-26T04:40:39.286763+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:32:33.977936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056498+0000","last_change":"2024-04-26T04:39:56.314940+0000","last_active":"2024-04-26T04:40:39.056498+0000","last_peered":"2024-04-26T04:40:39.056498+0000","last_clean":"2024-04-26T04:40:39.056498+0000","last_became_active":"2024-04-26T04:39:56.314651+0000","last_became_peered":"2024-04-26T04:39:56.314651+0000","last_unstale":"2024-04-26T04:40:39.056498+0000","last_undegraded":"2024-04-26T04:40:39.056498+0000","last_fullsized":"2024-04-26T04:40:39.056498+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:07:45.865637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057141+0000","last_change":"2024-04-26T04:39:56.291555+0000","last_active":"2024-04-26T04:40:39.057141+0000","last_peered":"2024-04-26T04:40:39.057141+0000","last_clean":"2024-04-26T04:40:39.057141+0000","last_became_active":"2024-04-26T04:39:56.291266+0000","last_became_peered":"2024-04-26T04:39:56.291266+0000","last_unstale":"2024-04-26T04:40:39.057141+0000","last_undegraded":"2024-04-26T04:40:39.057141+0000","last_fullsized":"2024-04-26T04:40:39.057141+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:50:56.830886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287185+0000","last_change":"2024-04-26T04:39:56.314624+0000","last_active":"2024-04-26T04:40:39.287185+0000","last_peered":"2024-04-26T04:40:39.287185+0000","last_clean":"2024-04-26T04:40:39.287185+0000","last_became_active":"2024-04-26T04:39:56.314332+0000","last_became_peered":"2024-04-26T04:39:56.314332+0000","last_unstale":"2024-04-26T04:40:39.287185+0000","last_undegraded":"2024-04-26T04:40:39.287185+0000","last_fullsized":"2024-04-26T04:40:39.287185+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:35:09.627440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053230+0000","last_change":"2024-04-26T04:39:56.299459+0000","last_active":"2024-04-26T04:40:39.053230+0000","last_peered":"2024-04-26T04:40:39.053230+0000","last_clean":"2024-04-26T04:40:39.053230+0000","last_became_active":"2024-04-26T04:39:56.299180+0000","last_became_peered":"2024-04-26T04:39:56.299180+0000","last_unstale":"2024-04-26T04:40:39.053230+0000","last_undegraded":"2024-04-26T04:40:39.053230+0000","last_fullsized":"2024-04-26T04:40:39.053230+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:51:01.512924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287301+0000","last_change":"2024-04-26T04:39:56.318500+0000","last_active":"2024-04-26T04:40:39.287301+0000","last_peered":"2024-04-26T04:40:39.287301+0000","last_clean":"2024-04-26T04:40:39.287301+0000","last_became_active":"2024-04-26T04:39:56.318211+0000","last_became_peered":"2024-04-26T04:39:56.318211+0000","last_unstale":"2024-04-26T04:40:39.287301+0000","last_undegraded":"2024-04-26T04:40:39.287301+0000","last_fullsized":"2024-04-26T04:40:39.287301+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:53:04.341594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053236+0000","last_change":"2024-04-26T04:39:56.297710+0000","last_active":"2024-04-26T04:40:39.053236+0000","last_peered":"2024-04-26T04:40:39.053236+0000","last_clean":"2024-04-26T04:40:39.053236+0000","last_became_active":"2024-04-26T04:39:56.297435+0000","last_became_peered":"2024-04-26T04:39:56.297435+0000","last_unstale":"2024-04-26T04:40:39.053236+0000","last_undegraded":"2024-04-26T04:40:39.053236+0000","last_fullsized":"2024-04-26T04:40:39.053236+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:37:34.665357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:40:40.088755+0000","last_change":"2024-04-26T04:40:40.088755+0000","last_active":"2024-04-26T04:40:40.088755+0000","last_peered":"2024-04-26T04:40:40.088755+0000","last_clean":"2024-04-26T04:40:40.088755+0000","last_became_active":"2024-04-26T04:40:40.088473+0000","last_became_peered":"2024-04-26T04:40:40.088473+0000","last_unstale":"2024-04-26T04:40:40.088755+0000","last_undegraded":"2024-04-26T04:40:40.088755+0000","last_fullsized":"2024-04-26T04:40:40.088755+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:41:44.433736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286618+0000","last_change":"2024-04-26T04:39:56.297829+0000","last_active":"2024-04-26T04:40:39.286618+0000","last_peered":"2024-04-26T04:40:39.286618+0000","last_clean":"2024-04-26T04:40:39.286618+0000","last_became_active":"2024-04-26T04:39:56.297539+0000","last_became_peered":"2024-04-26T04:39:56.297539+0000","last_unstale":"2024-04-26T04:40:39.286618+0000","last_undegraded":"2024-04-26T04:40:39.286618+0000","last_fullsized":"2024-04-26T04:40:39.286618+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:20:28.665493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287124+0000","last_change":"2024-04-26T04:39:56.315667+0000","last_active":"2024-04-26T04:40:39.287124+0000","last_peered":"2024-04-26T04:40:39.287124+0000","last_clean":"2024-04-26T04:40:39.287124+0000","last_became_active":"2024-04-26T04:39:56.315353+0000","last_became_peered":"2024-04-26T04:39:56.315353+0000","last_unstale":"2024-04-26T04:40:39.287124+0000","last_undegraded":"2024-04-26T04:40:39.287124+0000","last_fullsized":"2024-04-26T04:40:39.287124+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:45:00.659197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286647+0000","last_change":"2024-04-26T04:39:56.296410+0000","last_active":"2024-04-26T04:40:39.286647+0000","last_peered":"2024-04-26T04:40:39.286647+0000","last_clean":"2024-04-26T04:40:39.286647+0000","last_became_active":"2024-04-26T04:39:56.296116+0000","last_became_peered":"2024-04-26T04:39:56.296116+0000","last_unstale":"2024-04-26T04:40:39.286647+0000","last_undegraded":"2024-04-26T04:40:39.286647+0000","last_fullsized":"2024-04-26T04:40:39.286647+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:35:53.479201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056093+0000","last_change":"2024-04-26T04:38:54.064213+0000","last_active":"2024-04-26T04:40:39.056093+0000","last_peered":"2024-04-26T04:40:39.056093+0000","last_clean":"2024-04-26T04:40:39.056093+0000","last_became_active":"2024-04-26T04:38:54.063797+0000","last_became_peered":"2024-04-26T04:38:54.063797+0000","last_unstale":"2024-04-26T04:40:39.056093+0000","last_undegraded":"2024-04-26T04:40:39.056093+0000","last_fullsized":"2024-04-26T04:40:39.056093+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:21.370670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056073+0000","last_change":"2024-04-26T04:39:56.924091+0000","last_active":"2024-04-26T04:40:39.056073+0000","last_peered":"2024-04-26T04:40:39.056073+0000","last_clean":"2024-04-26T04:40:39.056073+0000","last_became_active":"2024-04-26T04:39:56.923807+0000","last_became_peered":"2024-04-26T04:39:56.923807+0000","last_unstale":"2024-04-26T04:40:39.056073+0000","last_undegraded":"2024-04-26T04:40:39.056073+0000","last_fullsized":"2024-04-26T04:40:39.056073+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:06:01.401463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059986+0000","last_change":"2024-04-26T04:38:54.530923+0000","last_active":"2024-04-26T04:40:39.059986+0000","last_peered":"2024-04-26T04:40:39.059986+0000","last_clean":"2024-04-26T04:40:39.059986+0000","last_became_active":"2024-04-26T04:38:54.530540+0000","last_became_peered":"2024-04-26T04:38:54.530540+0000","last_unstale":"2024-04-26T04:40:39.059986+0000","last_undegraded":"2024-04-26T04:40:39.059986+0000","last_fullsized":"2024-04-26T04:40:39.059986+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:28:56.675606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057127+0000","last_change":"2024-04-26T04:39:56.299210+0000","last_active":"2024-04-26T04:40:39.057127+0000","last_peered":"2024-04-26T04:40:39.057127+0000","last_clean":"2024-04-26T04:40:39.057127+0000","last_became_active":"2024-04-26T04:39:56.298905+0000","last_became_peered":"2024-04-26T04:39:56.298905+0000","last_unstale":"2024-04-26T04:40:39.057127+0000","last_undegraded":"2024-04-26T04:40:39.057127+0000","last_fullsized":"2024-04-26T04:40:39.057127+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:38:35.254187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286555+0000","last_change":"2024-04-26T04:39:56.923802+0000","last_active":"2024-04-26T04:40:39.286555+0000","last_peered":"2024-04-26T04:40:39.286555+0000","last_clean":"2024-04-26T04:40:39.286555+0000","last_became_active":"2024-04-26T04:39:56.923501+0000","last_became_peered":"2024-04-26T04:39:56.923501+0000","last_unstale":"2024-04-26T04:40:39.286555+0000","last_undegraded":"2024-04-26T04:40:39.286555+0000","last_fullsized":"2024-04-26T04:40:39.286555+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:00:46.110996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059431+0000","last_change":"2024-04-26T04:39:56.293821+0000","last_active":"2024-04-26T04:40:39.059431+0000","last_peered":"2024-04-26T04:40:39.059431+0000","last_clean":"2024-04-26T04:40:39.059431+0000","last_became_active":"2024-04-26T04:39:56.292175+0000","last_became_peered":"2024-04-26T04:39:56.292175+0000","last_unstale":"2024-04-26T04:40:39.059431+0000","last_undegraded":"2024-04-26T04:40:39.059431+0000","last_fullsized":"2024-04-26T04:40:39.059431+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:40:55.049762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057715+0000","last_change":"2024-04-26T04:38:54.530974+0000","last_active":"2024-04-26T04:40:39.057715+0000","last_peered":"2024-04-26T04:40:39.057715+0000","last_clean":"2024-04-26T04:40:39.057715+0000","last_became_active":"2024-04-26T04:38:54.530538+0000","last_became_peered":"2024-04-26T04:38:54.530538+0000","last_unstale":"2024-04-26T04:40:39.057715+0000","last_undegraded":"2024-04-26T04:40:39.057715+0000","last_fullsized":"2024-04-26T04:40:39.057715+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:40:04.608877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286690+0000","last_change":"2024-04-26T04:39:56.318781+0000","last_active":"2024-04-26T04:40:39.286690+0000","last_peered":"2024-04-26T04:40:39.286690+0000","last_clean":"2024-04-26T04:40:39.286690+0000","last_became_active":"2024-04-26T04:39:56.318414+0000","last_became_peered":"2024-04-26T04:39:56.318414+0000","last_unstale":"2024-04-26T04:40:39.286690+0000","last_undegraded":"2024-04-26T04:40:39.286690+0000","last_fullsized":"2024-04-26T04:40:39.286690+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:56:28.545804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286436+0000","last_change":"2024-04-26T04:39:56.298765+0000","last_active":"2024-04-26T04:40:39.286436+0000","last_peered":"2024-04-26T04:40:39.286436+0000","last_clean":"2024-04-26T04:40:39.286436+0000","last_became_active":"2024-04-26T04:39:56.298496+0000","last_became_peered":"2024-04-26T04:39:56.298496+0000","last_unstale":"2024-04-26T04:40:39.286436+0000","last_undegraded":"2024-04-26T04:40:39.286436+0000","last_fullsized":"2024-04-26T04:40:39.286436+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:34:31.926911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057383+0000","last_change":"2024-04-26T04:38:54.065770+0000","last_active":"2024-04-26T04:40:39.057383+0000","last_peered":"2024-04-26T04:40:39.057383+0000","last_clean":"2024-04-26T04:40:39.057383+0000","last_became_active":"2024-04-26T04:38:54.065409+0000","last_became_peered":"2024-04-26T04:38:54.065409+0000","last_unstale":"2024-04-26T04:40:39.057383+0000","last_undegraded":"2024-04-26T04:40:39.057383+0000","last_fullsized":"2024-04-26T04:40:39.057383+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:56.230650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.055488+0000","last_change":"2024-04-26T04:39:56.923636+0000","last_active":"2024-04-26T04:40:39.055488+0000","last_peered":"2024-04-26T04:40:39.055488+0000","last_clean":"2024-04-26T04:40:39.055488+0000","last_became_active":"2024-04-26T04:39:56.923324+0000","last_became_peered":"2024-04-26T04:39:56.923324+0000","last_unstale":"2024-04-26T04:40:39.055488+0000","last_undegraded":"2024-04-26T04:40:39.055488+0000","last_fullsized":"2024-04-26T04:40:39.055488+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:05:54.903060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056358+0000","last_change":"2024-04-26T04:39:56.291091+0000","last_active":"2024-04-26T04:40:39.056358+0000","last_peered":"2024-04-26T04:40:39.056358+0000","last_clean":"2024-04-26T04:40:39.056358+0000","last_became_active":"2024-04-26T04:39:56.290764+0000","last_became_peered":"2024-04-26T04:39:56.290764+0000","last_unstale":"2024-04-26T04:40:39.056358+0000","last_undegraded":"2024-04-26T04:40:39.056358+0000","last_fullsized":"2024-04-26T04:40:39.056358+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:40:14.466242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057304+0000","last_change":"2024-04-26T04:38:11.717965+0000","last_active":"2024-04-26T04:40:39.057304+0000","last_peered":"2024-04-26T04:40:39.057304+0000","last_clean":"2024-04-26T04:40:39.057304+0000","last_became_active":"2024-04-26T04:38:11.717490+0000","last_became_peered":"2024-04-26T04:38:11.717490+0000","last_unstale":"2024-04-26T04:40:39.057304+0000","last_undegraded":"2024-04-26T04:40:39.057304+0000","last_fullsized":"2024-04-26T04:40:39.057304+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_clean_scrub_stamp":"2024-04-26T04:36:10.749042+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:18:38.216134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053025+0000","last_change":"2024-04-26T04:38:54.063499+0000","last_active":"2024-04-26T04:40:39.053025+0000","last_peered":"2024-04-26T04:40:39.053025+0000","last_clean":"2024-04-26T04:40:39.053025+0000","last_became_active":"2024-04-26T04:38:54.063035+0000","last_became_peered":"2024-04-26T04:38:54.063035+0000","last_unstale":"2024-04-26T04:40:39.053025+0000","last_undegraded":"2024-04-26T04:40:39.053025+0000","last_fullsized":"2024-04-26T04:40:39.053025+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:20.279953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060170+0000","last_change":"2024-04-26T04:39:56.323989+0000","last_active":"2024-04-26T04:40:39.060170+0000","last_peered":"2024-04-26T04:40:39.060170+0000","last_clean":"2024-04-26T04:40:39.060170+0000","last_became_active":"2024-04-26T04:39:56.322916+0000","last_became_peered":"2024-04-26T04:39:56.322916+0000","last_unstale":"2024-04-26T04:40:39.060170+0000","last_undegraded":"2024-04-26T04:40:39.060170+0000","last_fullsized":"2024-04-26T04:40:39.060170+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:44:47.533970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059343+0000","last_change":"2024-04-26T04:39:56.293415+0000","last_active":"2024-04-26T04:40:39.059343+0000","last_peered":"2024-04-26T04:40:39.059343+0000","last_clean":"2024-04-26T04:40:39.059343+0000","last_became_active":"2024-04-26T04:39:56.291575+0000","last_became_peered":"2024-04-26T04:39:56.291575+0000","last_unstale":"2024-04-26T04:40:39.059343+0000","last_undegraded":"2024-04-26T04:40:39.059343+0000","last_fullsized":"2024-04-26T04:40:39.059343+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:20.825043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056244+0000","last_change":"2024-04-26T04:38:54.066302+0000","last_active":"2024-04-26T04:40:39.056244+0000","last_peered":"2024-04-26T04:40:39.056244+0000","last_clean":"2024-04-26T04:40:39.056244+0000","last_became_active":"2024-04-26T04:38:54.065974+0000","last_became_peered":"2024-04-26T04:38:54.065974+0000","last_unstale":"2024-04-26T04:40:39.056244+0000","last_undegraded":"2024-04-26T04:40:39.056244+0000","last_fullsized":"2024-04-26T04:40:39.056244+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:27:22.809404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059347+0000","last_change":"2024-04-26T04:39:56.324186+0000","last_active":"2024-04-26T04:40:39.059347+0000","last_peered":"2024-04-26T04:40:39.059347+0000","last_clean":"2024-04-26T04:40:39.059347+0000","last_became_active":"2024-04-26T04:39:56.323936+0000","last_became_peered":"2024-04-26T04:39:56.323936+0000","last_unstale":"2024-04-26T04:40:39.059347+0000","last_undegraded":"2024-04-26T04:40:39.059347+0000","last_fullsized":"2024-04-26T04:40:39.059347+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:56:56.391281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287242+0000","last_change":"2024-04-26T04:39:56.292521+0000","last_active":"2024-04-26T04:40:39.287242+0000","last_peered":"2024-04-26T04:40:39.287242+0000","last_clean":"2024-04-26T04:40:39.287242+0000","last_became_active":"2024-04-26T04:39:56.292247+0000","last_became_peered":"2024-04-26T04:39:56.292247+0000","last_unstale":"2024-04-26T04:40:39.287242+0000","last_undegraded":"2024-04-26T04:40:39.287242+0000","last_fullsized":"2024-04-26T04:40:39.287242+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:47:20.168803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.052978+0000","last_change":"2024-04-26T04:38:54.064959+0000","last_active":"2024-04-26T04:40:39.052978+0000","last_peered":"2024-04-26T04:40:39.052978+0000","last_clean":"2024-04-26T04:40:39.052978+0000","last_became_active":"2024-04-26T04:38:54.064530+0000","last_became_peered":"2024-04-26T04:38:54.064530+0000","last_unstale":"2024-04-26T04:40:39.052978+0000","last_undegraded":"2024-04-26T04:40:39.052978+0000","last_fullsized":"2024-04-26T04:40:39.052978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:20.559351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060106+0000","last_change":"2024-04-26T04:39:56.322825+0000","last_active":"2024-04-26T04:40:39.060106+0000","last_peered":"2024-04-26T04:40:39.060106+0000","last_clean":"2024-04-26T04:40:39.060106+0000","last_became_active":"2024-04-26T04:39:56.322517+0000","last_became_peered":"2024-04-26T04:39:56.322517+0000","last_unstale":"2024-04-26T04:40:39.060106+0000","last_undegraded":"2024-04-26T04:40:39.060106+0000","last_fullsized":"2024-04-26T04:40:39.060106+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:50:08.121628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053199+0000","last_change":"2024-04-26T04:39:56.292684+0000","last_active":"2024-04-26T04:40:39.053199+0000","last_peered":"2024-04-26T04:40:39.053199+0000","last_clean":"2024-04-26T04:40:39.053199+0000","last_became_active":"2024-04-26T04:39:56.291835+0000","last_became_peered":"2024-04-26T04:39:56.291835+0000","last_unstale":"2024-04-26T04:40:39.053199+0000","last_undegraded":"2024-04-26T04:40:39.053199+0000","last_fullsized":"2024-04-26T04:40:39.053199+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:27:12.540494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059265+0000","last_change":"2024-04-26T04:38:54.065701+0000","last_active":"2024-04-26T04:40:39.059265+0000","last_peered":"2024-04-26T04:40:39.059265+0000","last_clean":"2024-04-26T04:40:39.059265+0000","last_became_active":"2024-04-26T04:38:54.065370+0000","last_became_peered":"2024-04-26T04:38:54.065370+0000","last_unstale":"2024-04-26T04:40:39.059265+0000","last_undegraded":"2024-04-26T04:40:39.059265+0000","last_fullsized":"2024-04-26T04:40:39.059265+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:38.001692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060257+0000","last_change":"2024-04-26T04:39:56.324150+0000","last_active":"2024-04-26T04:40:39.060257+0000","last_peered":"2024-04-26T04:40:39.060257+0000","last_clean":"2024-04-26T04:40:39.060257+0000","last_became_active":"2024-04-26T04:39:56.323725+0000","last_became_peered":"2024-04-26T04:39:56.323725+0000","last_unstale":"2024-04-26T04:40:39.060257+0000","last_undegraded":"2024-04-26T04:40:39.060257+0000","last_fullsized":"2024-04-26T04:40:39.060257+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:33:11.245827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056183+0000","last_change":"2024-04-26T04:39:56.301577+0000","last_active":"2024-04-26T04:40:39.056183+0000","last_peered":"2024-04-26T04:40:39.056183+0000","last_clean":"2024-04-26T04:40:39.056183+0000","last_became_active":"2024-04-26T04:39:56.301060+0000","last_became_peered":"2024-04-26T04:39:56.301060+0000","last_unstale":"2024-04-26T04:40:39.056183+0000","last_undegraded":"2024-04-26T04:40:39.056183+0000","last_fullsized":"2024-04-26T04:40:39.056183+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:31:39.931897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287614+0000","last_change":"2024-04-26T04:39:56.316678+0000","last_active":"2024-04-26T04:40:39.287614+0000","last_peered":"2024-04-26T04:40:39.287614+0000","last_clean":"2024-04-26T04:40:39.287614+0000","last_became_active":"2024-04-26T04:39:56.316374+0000","last_became_peered":"2024-04-26T04:39:56.316374+0000","last_unstale":"2024-04-26T04:40:39.287614+0000","last_undegraded":"2024-04-26T04:40:39.287614+0000","last_fullsized":"2024-04-26T04:40:39.287614+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:50:38.289489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287182+0000","last_change":"2024-04-26T04:39:56.298990+0000","last_active":"2024-04-26T04:40:39.287182+0000","last_peered":"2024-04-26T04:40:39.287182+0000","last_clean":"2024-04-26T04:40:39.287182+0000","last_became_active":"2024-04-26T04:39:56.298660+0000","last_became_peered":"2024-04-26T04:39:56.298660+0000","last_unstale":"2024-04-26T04:40:39.287182+0000","last_undegraded":"2024-04-26T04:40:39.287182+0000","last_fullsized":"2024-04-26T04:40:39.287182+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:48:46.486720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287561+0000","last_change":"2024-04-26T04:39:56.318571+0000","last_active":"2024-04-26T04:40:39.287561+0000","last_peered":"2024-04-26T04:40:39.287561+0000","last_clean":"2024-04-26T04:40:39.287561+0000","last_became_active":"2024-04-26T04:39:56.318273+0000","last_became_peered":"2024-04-26T04:39:56.318273+0000","last_unstale":"2024-04-26T04:40:39.287561+0000","last_undegraded":"2024-04-26T04:40:39.287561+0000","last_fullsized":"2024-04-26T04:40:39.287561+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:54:58.284911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.055585+0000","last_change":"2024-04-26T04:39:56.294993+0000","last_active":"2024-04-26T04:40:39.055585+0000","last_peered":"2024-04-26T04:40:39.055585+0000","last_clean":"2024-04-26T04:40:39.055585+0000","last_became_active":"2024-04-26T04:39:56.294723+0000","last_became_peered":"2024-04-26T04:39:56.294723+0000","last_unstale":"2024-04-26T04:40:39.055585+0000","last_undegraded":"2024-04-26T04:40:39.055585+0000","last_fullsized":"2024-04-26T04:40:39.055585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:01:25.922226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287314+0000","last_change":"2024-04-26T04:39:56.315908+0000","last_active":"2024-04-26T04:40:39.287314+0000","last_peered":"2024-04-26T04:40:39.287314+0000","last_clean":"2024-04-26T04:40:39.287314+0000","last_became_active":"2024-04-26T04:39:56.315616+0000","last_became_peered":"2024-04-26T04:39:56.315616+0000","last_unstale":"2024-04-26T04:40:39.287314+0000","last_undegraded":"2024-04-26T04:40:39.287314+0000","last_fullsized":"2024-04-26T04:40:39.287314+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:23:47.585151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056864+0000","last_change":"2024-04-26T04:39:56.298264+0000","last_active":"2024-04-26T04:40:39.056864+0000","last_peered":"2024-04-26T04:40:39.056864+0000","last_clean":"2024-04-26T04:40:39.056864+0000","last_became_active":"2024-04-26T04:39:56.297945+0000","last_became_peered":"2024-04-26T04:39:56.297945+0000","last_unstale":"2024-04-26T04:40:39.056864+0000","last_undegraded":"2024-04-26T04:40:39.056864+0000","last_fullsized":"2024-04-26T04:40:39.056864+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:15:20.706729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286467+0000","last_change":"2024-04-26T04:39:56.319985+0000","last_active":"2024-04-26T04:40:39.286467+0000","last_peered":"2024-04-26T04:40:39.286467+0000","last_clean":"2024-04-26T04:40:39.286467+0000","last_became_active":"2024-04-26T04:39:56.319677+0000","last_became_peered":"2024-04-26T04:39:56.319677+0000","last_unstale":"2024-04-26T04:40:39.286467+0000","last_undegraded":"2024-04-26T04:40:39.286467+0000","last_fullsized":"2024-04-26T04:40:39.286467+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:06:56.732447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053141+0000","last_change":"2024-04-26T04:39:56.295923+0000","last_active":"2024-04-26T04:40:39.053141+0000","last_peered":"2024-04-26T04:40:39.053141+0000","last_clean":"2024-04-26T04:40:39.053141+0000","last_became_active":"2024-04-26T04:39:56.295537+0000","last_became_peered":"2024-04-26T04:39:56.295537+0000","last_unstale":"2024-04-26T04:40:39.053141+0000","last_undegraded":"2024-04-26T04:40:39.053141+0000","last_fullsized":"2024-04-26T04:40:39.053141+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:25:33.453364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:40:40.450338+0000","last_change":"2024-04-26T04:40:40.450338+0000","last_active":"2024-04-26T04:40:40.450338+0000","last_peered":"2024-04-26T04:40:40.450338+0000","last_clean":"2024-04-26T04:40:40.450338+0000","last_became_active":"2024-04-26T04:40:40.449961+0000","last_became_peered":"2024-04-26T04:40:40.449961+0000","last_unstale":"2024-04-26T04:40:40.450338+0000","last_undegraded":"2024-04-26T04:40:40.450338+0000","last_fullsized":"2024-04-26T04:40:40.450338+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:06:03.706318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057825+0000","last_change":"2024-04-26T04:39:56.300360+0000","last_active":"2024-04-26T04:40:39.057825+0000","last_peered":"2024-04-26T04:40:39.057825+0000","last_clean":"2024-04-26T04:40:39.057825+0000","last_became_active":"2024-04-26T04:39:56.300067+0000","last_became_peered":"2024-04-26T04:39:56.300067+0000","last_unstale":"2024-04-26T04:40:39.057825+0000","last_undegraded":"2024-04-26T04:40:39.057825+0000","last_fullsized":"2024-04-26T04:40:39.057825+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:53:28.753150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287044+0000","last_change":"2024-04-26T04:39:56.318542+0000","last_active":"2024-04-26T04:40:39.287044+0000","last_peered":"2024-04-26T04:40:39.287044+0000","last_clean":"2024-04-26T04:40:39.287044+0000","last_became_active":"2024-04-26T04:39:56.318294+0000","last_became_peered":"2024-04-26T04:39:56.318294+0000","last_unstale":"2024-04-26T04:40:39.287044+0000","last_undegraded":"2024-04-26T04:40:39.287044+0000","last_fullsized":"2024-04-26T04:40:39.287044+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:54:41.940567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057699+0000","last_change":"2024-04-26T04:39:56.304876+0000","last_active":"2024-04-26T04:40:39.057699+0000","last_peered":"2024-04-26T04:40:39.057699+0000","last_clean":"2024-04-26T04:40:39.057699+0000","last_became_active":"2024-04-26T04:39:56.304597+0000","last_became_peered":"2024-04-26T04:39:56.304597+0000","last_unstale":"2024-04-26T04:40:39.057699+0000","last_undegraded":"2024-04-26T04:40:39.057699+0000","last_fullsized":"2024-04-26T04:40:39.057699+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:16:18.446961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.052921+0000","last_change":"2024-04-26T04:39:56.925632+0000","last_active":"2024-04-26T04:40:39.052921+0000","last_peered":"2024-04-26T04:40:39.052921+0000","last_clean":"2024-04-26T04:40:39.052921+0000","last_became_active":"2024-04-26T04:39:56.925327+0000","last_became_peered":"2024-04-26T04:39:56.925327+0000","last_unstale":"2024-04-26T04:40:39.052921+0000","last_undegraded":"2024-04-26T04:40:39.052921+0000","last_fullsized":"2024-04-26T04:40:39.052921+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:29:59.594965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.058017+0000","last_change":"2024-04-26T04:39:56.304531+0000","last_active":"2024-04-26T04:40:39.058017+0000","last_peered":"2024-04-26T04:40:39.058017+0000","last_clean":"2024-04-26T04:40:39.058017+0000","last_became_active":"2024-04-26T04:39:56.304209+0000","last_became_peered":"2024-04-26T04:39:56.304209+0000","last_unstale":"2024-04-26T04:40:39.058017+0000","last_undegraded":"2024-04-26T04:40:39.058017+0000","last_fullsized":"2024-04-26T04:40:39.058017+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:01:36.403239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056872+0000","last_change":"2024-04-26T04:39:56.925049+0000","last_active":"2024-04-26T04:40:39.056872+0000","last_peered":"2024-04-26T04:40:39.056872+0000","last_clean":"2024-04-26T04:40:39.056872+0000","last_became_active":"2024-04-26T04:39:56.924725+0000","last_became_peered":"2024-04-26T04:39:56.924725+0000","last_unstale":"2024-04-26T04:40:39.056872+0000","last_undegraded":"2024-04-26T04:40:39.056872+0000","last_fullsized":"2024-04-26T04:40:39.056872+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:46:34.564332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.055359+0000","last_change":"2024-04-26T04:39:56.293952+0000","last_active":"2024-04-26T04:40:39.055359+0000","last_peered":"2024-04-26T04:40:39.055359+0000","last_clean":"2024-04-26T04:40:39.055359+0000","last_became_active":"2024-04-26T04:39:56.293651+0000","last_became_peered":"2024-04-26T04:39:56.293651+0000","last_unstale":"2024-04-26T04:40:39.055359+0000","last_undegraded":"2024-04-26T04:40:39.055359+0000","last_fullsized":"2024-04-26T04:40:39.055359+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:13:19.299051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286549+0000","last_change":"2024-04-26T04:39:56.923289+0000","last_active":"2024-04-26T04:40:39.286549+0000","last_peered":"2024-04-26T04:40:39.286549+0000","last_clean":"2024-04-26T04:40:39.286549+0000","last_became_active":"2024-04-26T04:39:56.922986+0000","last_became_peered":"2024-04-26T04:39:56.922986+0000","last_unstale":"2024-04-26T04:40:39.286549+0000","last_undegraded":"2024-04-26T04:40:39.286549+0000","last_fullsized":"2024-04-26T04:40:39.286549+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:36:03.904766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057317+0000","last_change":"2024-04-26T04:39:56.302393+0000","last_active":"2024-04-26T04:40:39.057317+0000","last_peered":"2024-04-26T04:40:39.057317+0000","last_clean":"2024-04-26T04:40:39.057317+0000","last_became_active":"2024-04-26T04:39:56.302116+0000","last_became_peered":"2024-04-26T04:39:56.302116+0000","last_unstale":"2024-04-26T04:40:39.057317+0000","last_undegraded":"2024-04-26T04:40:39.057317+0000","last_fullsized":"2024-04-26T04:40:39.057317+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:28:13.475580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053118+0000","last_change":"2024-04-26T04:39:56.924653+0000","last_active":"2024-04-26T04:40:39.053118+0000","last_peered":"2024-04-26T04:40:39.053118+0000","last_clean":"2024-04-26T04:40:39.053118+0000","last_became_active":"2024-04-26T04:39:56.924283+0000","last_became_peered":"2024-04-26T04:39:56.924283+0000","last_unstale":"2024-04-26T04:40:39.053118+0000","last_undegraded":"2024-04-26T04:40:39.053118+0000","last_fullsized":"2024-04-26T04:40:39.053118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:07:19.121146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286438+0000","last_change":"2024-04-26T04:39:56.300183+0000","last_active":"2024-04-26T04:40:39.286438+0000","last_peered":"2024-04-26T04:40:39.286438+0000","last_clean":"2024-04-26T04:40:39.286438+0000","last_became_active":"2024-04-26T04:39:56.299703+0000","last_became_peered":"2024-04-26T04:39:56.299703+0000","last_unstale":"2024-04-26T04:40:39.286438+0000","last_undegraded":"2024-04-26T04:40:39.286438+0000","last_fullsized":"2024-04-26T04:40:39.286438+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:33:28.080217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057095+0000","last_change":"2024-04-26T04:39:56.922951+0000","last_active":"2024-04-26T04:40:39.057095+0000","last_peered":"2024-04-26T04:40:39.057095+0000","last_clean":"2024-04-26T04:40:39.057095+0000","last_became_active":"2024-04-26T04:39:56.922592+0000","last_became_peered":"2024-04-26T04:39:56.922592+0000","last_unstale":"2024-04-26T04:40:39.057095+0000","last_undegraded":"2024-04-26T04:40:39.057095+0000","last_fullsized":"2024-04-26T04:40:39.057095+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:07:26.648362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057952+0000","last_change":"2024-04-26T04:39:56.304345+0000","last_active":"2024-04-26T04:40:39.057952+0000","last_peered":"2024-04-26T04:40:39.057952+0000","last_clean":"2024-04-26T04:40:39.057952+0000","last_became_active":"2024-04-26T04:39:56.304040+0000","last_became_peered":"2024-04-26T04:39:56.304040+0000","last_unstale":"2024-04-26T04:40:39.057952+0000","last_undegraded":"2024-04-26T04:40:39.057952+0000","last_fullsized":"2024-04-26T04:40:39.057952+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:37:54.252912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287204+0000","last_change":"2024-04-26T04:39:56.318742+0000","last_active":"2024-04-26T04:40:39.287204+0000","last_peered":"2024-04-26T04:40:39.287204+0000","last_clean":"2024-04-26T04:40:39.287204+0000","last_became_active":"2024-04-26T04:39:56.318441+0000","last_became_peered":"2024-04-26T04:39:56.318441+0000","last_unstale":"2024-04-26T04:40:39.287204+0000","last_undegraded":"2024-04-26T04:40:39.287204+0000","last_fullsized":"2024-04-26T04:40:39.287204+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:55:08.709896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287391+0000","last_change":"2024-04-26T04:39:56.300576+0000","last_active":"2024-04-26T04:40:39.287391+0000","last_peered":"2024-04-26T04:40:39.287391+0000","last_clean":"2024-04-26T04:40:39.287391+0000","last_became_active":"2024-04-26T04:39:56.299260+0000","last_became_peered":"2024-04-26T04:39:56.299260+0000","last_unstale":"2024-04-26T04:40:39.287391+0000","last_undegraded":"2024-04-26T04:40:39.287391+0000","last_fullsized":"2024-04-26T04:40:39.287391+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:26.586898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060169+0000","last_change":"2024-04-26T04:39:56.323585+0000","last_active":"2024-04-26T04:40:39.060169+0000","last_peered":"2024-04-26T04:40:39.060169+0000","last_clean":"2024-04-26T04:40:39.060169+0000","last_became_active":"2024-04-26T04:39:56.323322+0000","last_became_peered":"2024-04-26T04:39:56.323322+0000","last_unstale":"2024-04-26T04:40:39.060169+0000","last_undegraded":"2024-04-26T04:40:39.060169+0000","last_fullsized":"2024-04-26T04:40:39.060169+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:08:11.656124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.058059+0000","last_change":"2024-04-26T04:39:56.304338+0000","last_active":"2024-04-26T04:40:39.058059+0000","last_peered":"2024-04-26T04:40:39.058059+0000","last_clean":"2024-04-26T04:40:39.058059+0000","last_became_active":"2024-04-26T04:39:56.304033+0000","last_became_peered":"2024-04-26T04:39:56.304033+0000","last_unstale":"2024-04-26T04:40:39.058059+0000","last_undegraded":"2024-04-26T04:40:39.058059+0000","last_fullsized":"2024-04-26T04:40:39.058059+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:57.154652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056308+0000","last_change":"2024-04-26T04:39:56.923328+0000","last_active":"2024-04-26T04:40:39.056308+0000","last_peered":"2024-04-26T04:40:39.056308+0000","last_clean":"2024-04-26T04:40:39.056308+0000","last_became_active":"2024-04-26T04:39:56.923032+0000","last_became_peered":"2024-04-26T04:39:56.923032+0000","last_unstale":"2024-04-26T04:40:39.056308+0000","last_undegraded":"2024-04-26T04:40:39.056308+0000","last_fullsized":"2024-04-26T04:40:39.056308+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:34:11.959235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053296+0000","last_change":"2024-04-26T04:39:56.297527+0000","last_active":"2024-04-26T04:40:39.053296+0000","last_peered":"2024-04-26T04:40:39.053296+0000","last_clean":"2024-04-26T04:40:39.053296+0000","last_became_active":"2024-04-26T04:39:56.297235+0000","last_became_peered":"2024-04-26T04:39:56.297235+0000","last_unstale":"2024-04-26T04:40:39.053296+0000","last_undegraded":"2024-04-26T04:40:39.053296+0000","last_fullsized":"2024-04-26T04:40:39.053296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:53.928583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056264+0000","last_change":"2024-04-26T04:39:56.321557+0000","last_active":"2024-04-26T04:40:39.056264+0000","last_peered":"2024-04-26T04:40:39.056264+0000","last_clean":"2024-04-26T04:40:39.056264+0000","last_became_active":"2024-04-26T04:39:56.321228+0000","last_became_peered":"2024-04-26T04:39:56.321228+0000","last_unstale":"2024-04-26T04:40:39.056264+0000","last_undegraded":"2024-04-26T04:40:39.056264+0000","last_fullsized":"2024-04-26T04:40:39.056264+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:18:27.526771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286710+0000","last_change":"2024-04-26T04:39:56.299978+0000","last_active":"2024-04-26T04:40:39.286710+0000","last_peered":"2024-04-26T04:40:39.286710+0000","last_clean":"2024-04-26T04:40:39.286710+0000","last_became_active":"2024-04-26T04:39:56.299603+0000","last_became_peered":"2024-04-26T04:39:56.299603+0000","last_unstale":"2024-04-26T04:40:39.286710+0000","last_undegraded":"2024-04-26T04:40:39.286710+0000","last_fullsized":"2024-04-26T04:40:39.286710+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:03:24.553191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:40:40.088501+0000","last_change":"2024-04-26T04:40:40.088501+0000","last_active":"2024-04-26T04:40:40.088501+0000","last_peered":"2024-04-26T04:40:40.088501+0000","last_clean":"2024-04-26T04:40:40.088501+0000","last_became_active":"2024-04-26T04:40:40.088060+0000","last_became_peered":"2024-04-26T04:40:40.088060+0000","last_unstale":"2024-04-26T04:40:40.088501+0000","last_undegraded":"2024-04-26T04:40:40.088501+0000","last_fullsized":"2024-04-26T04:40:40.088501+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:04:13.569242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056359+0000","last_change":"2024-04-26T04:39:56.292587+0000","last_active":"2024-04-26T04:40:39.056359+0000","last_peered":"2024-04-26T04:40:39.056359+0000","last_clean":"2024-04-26T04:40:39.056359+0000","last_became_active":"2024-04-26T04:39:56.292332+0000","last_became_peered":"2024-04-26T04:39:56.292332+0000","last_unstale":"2024-04-26T04:40:39.056359+0000","last_undegraded":"2024-04-26T04:40:39.056359+0000","last_fullsized":"2024-04-26T04:40:39.056359+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:19:26.474036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286602+0000","last_change":"2024-04-26T04:39:56.923162+0000","last_active":"2024-04-26T04:40:39.286602+0000","last_peered":"2024-04-26T04:40:39.286602+0000","last_clean":"2024-04-26T04:40:39.286602+0000","last_became_active":"2024-04-26T04:39:56.922828+0000","last_became_peered":"2024-04-26T04:39:56.922828+0000","last_unstale":"2024-04-26T04:40:39.286602+0000","last_undegraded":"2024-04-26T04:40:39.286602+0000","last_fullsized":"2024-04-26T04:40:39.286602+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:20:11.342915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059620+0000","last_change":"2024-04-26T04:39:56.294919+0000","last_active":"2024-04-26T04:40:39.059620+0000","last_peered":"2024-04-26T04:40:39.059620+0000","last_clean":"2024-04-26T04:40:39.059620+0000","last_became_active":"2024-04-26T04:39:56.294617+0000","last_became_peered":"2024-04-26T04:39:56.294617+0000","last_unstale":"2024-04-26T04:40:39.059620+0000","last_undegraded":"2024-04-26T04:40:39.059620+0000","last_fullsized":"2024-04-26T04:40:39.059620+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:22:16.623574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056470+0000","last_change":"2024-04-26T04:39:56.320548+0000","last_active":"2024-04-26T04:40:39.056470+0000","last_peered":"2024-04-26T04:40:39.056470+0000","last_clean":"2024-04-26T04:40:39.056470+0000","last_became_active":"2024-04-26T04:39:56.320269+0000","last_became_peered":"2024-04-26T04:39:56.320269+0000","last_unstale":"2024-04-26T04:40:39.056470+0000","last_undegraded":"2024-04-26T04:40:39.056470+0000","last_fullsized":"2024-04-26T04:40:39.056470+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:17:07.659869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287058+0000","last_change":"2024-04-26T04:39:56.293361+0000","last_active":"2024-04-26T04:40:39.287058+0000","last_peered":"2024-04-26T04:40:39.287058+0000","last_clean":"2024-04-26T04:40:39.287058+0000","last_became_active":"2024-04-26T04:39:56.293064+0000","last_became_peered":"2024-04-26T04:39:56.293064+0000","last_unstale":"2024-04-26T04:40:39.287058+0000","last_undegraded":"2024-04-26T04:40:39.287058+0000","last_fullsized":"2024-04-26T04:40:39.287058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:57:47.052446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201314,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":901155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.40899999999999997,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.372}]},{"osd":1,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.53100000000000003,"15min":0.52300000000000002},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.44900000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53300000000000003,"15min":0.52700000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.65300000000000002,"15min":0.60299999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.9450000000000003,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":0.53100000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.85599999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.51000000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.51600000000000001,"15min":0.50700000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85599999999999998,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.67500000000000004}]},{"osd":5,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.51900000000000002,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.60599999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.69799999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462951,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":901547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77900000000000003,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.53200000000000003,"15min":0.52900000000000003},"min":{"1min":0.35399999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.63200000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.442,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76800000000000002,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.68600000000000005}]},{"osd":2,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.376,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.54600000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.58199999999999996}]},{"osd":3,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.53100000000000003,"15min":0.53200000000000003},"min":{"1min":0.35199999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57399999999999995,"15min":0.57299999999999995},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65900000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.47399999999999998,"15min":0.46400000000000002},"min":{"1min":0.374,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.5,"15min":0.48899999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.42399999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.48699999999999999,"15min":0.47799999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.78900000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.50900000000000001,"15min":0.497},"min":{"1min":0.34000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53400000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.56299999999999994,"15min":0.55800000000000005},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.79600000000000004,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.497},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79200000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.55000000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":309292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.373,"15min":0.33300000000000002},"min":{"1min":0.22900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.58199999999999996,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.35999999999999999},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.46800000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.42099999999999999,"15min":0.41599999999999998},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.48999999999999999,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.71399999999999997,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.436}]},{"osd":3,"last update":"Fri Apr 26 04:40:40 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.47099999999999997,"15min":0.48999999999999999},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61199999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.47199999999999998,"15min":0.498},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61399999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.34200000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.61899999999999999,"15min":0.627},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.76500000000000001,"5min":1.151,"15min":1.151},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.59399999999999997,"15min":0.60999999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.69999999999999996,"5min":1,"15min":1},"last":0.51900000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:40:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60899999999999999,"15min":0.61099999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.93500000000000005,"5min":1.133,"15min":1.133},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.623,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.78000000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.58599999999999997}]},{"osd":6,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.64500000000000002,"15min":0.65100000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.73299999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.63800000000000001,"15min":0.65000000000000002},"min":{"1min":0.46300000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.76200000000000001,"5min":1.069,"15min":1.069},"last":0.871},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.628,"15min":0.63800000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68200000000000005,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.70799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1314816,"data_stored":900517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:39:55 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36199999999999999,"15min":0.34799999999999998},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.56299999999999994,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.309},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40500000000000003,"15min":0.38200000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.36799999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:40:13 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.441,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.42099999999999999,"15min":0.42899999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58599999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.34499999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:40:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.46300000000000002,"15min":0.47299999999999998},"min":{"1min":0.33700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70699999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.39300000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84099999999999997,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58999999999999997,"15min":0.59399999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.83999999999999997,"5min":0.872,"15min":0.872},"last":0.56799999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:40:22 2024","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":0.81399999999999995,"15min":0.68600000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":15.811999999999999,"5min":15.811999999999999,"15min":15.811999999999999},"last":0.436},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.62,"15min":0.626},"min":{"1min":0.44800000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.60599999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.81399999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.65200000000000002,"15min":0.65600000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.5}]},{"osd":7,"last update":"Fri Apr 26 04:40:13 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59399999999999997,"15min":0.58799999999999997},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65400000000000003,"15min":0.65300000000000002},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.48199999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":310161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.48799999999999999,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.41399999999999998,"15min":0.40999999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.51700000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.26500000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.42699999999999999,"15min":0.433},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53500000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.441,"15min":0.443},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.496,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.318}]},{"osd":3,"last update":"Fri Apr 26 04:39:45 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.47099999999999997,"15min":0.48099999999999998},"min":{"1min":0.32000000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.65100000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.49299999999999999,"15min":0.5},"min":{"1min":0.33900000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.36499999999999999}]},{"osd":4,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58999999999999997,"15min":0.60199999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83499999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.64400000000000002,"15min":0.65200000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.504}]},{"osd":5,"last update":"Fri Apr 26 04:40:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.61399999999999999,"15min":0.63200000000000001},"min":{"1min":0.41499999999999998,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83999999999999997,"5min":0.999,"15min":0.999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57599999999999996,"15min":0.59599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.82099999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.49099999999999999}]},{"osd":6,"last update":"Fri Apr 26 04:39:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.61799999999999999,"15min":0.623},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.71999999999999997,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.63700000000000001,"15min":0.64400000000000002},"min":{"1min":0.45000000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.54100000000000004}]},{"osd":7,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.58399999999999996,"15min":0.58699999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.75,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.51400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":309647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:43 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.40300000000000002,"15min":0.40699999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.63500000000000001,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.22700000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.39900000000000002,"15min":0.39800000000000002},"min":{"1min":0.29099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.60799999999999998,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.32200000000000001}]},{"osd":1,"last update":"Fri Apr 26 04:40:43 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.41399999999999998,"15min":0.40899999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.79100000000000004,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.48399999999999999,"15min":0.50600000000000001},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.77900000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.35799999999999998}]},{"osd":2,"last update":"Fri Apr 26 04:40:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.45000000000000001,"15min":0.44600000000000001},"min":{"1min":0.27800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.68400000000000005,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.442,"15min":0.46400000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.25}]},{"osd":4,"last update":"Fri Apr 26 04:39:58 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58999999999999997,"15min":0.58399999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.89500000000000002,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.627,"15min":0.65000000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77000000000000002,"5min":1.034,"15min":1.034},"last":0.42399999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:40:29 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.46300000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.628,"15min":0.63900000000000001},"min":{"1min":0.41299999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80400000000000005,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.53000000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:39:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65800000000000003,"15min":0.66600000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.94899999999999995,"5min":0.998,"15min":0.998},"last":0.434},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.48999999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:40:10 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61799999999999999,"15min":0.61599999999999999},"min":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.64800000000000002,"15min":0.65100000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.96299999999999997,"5min":1.014,"15min":1.014},"last":0.54000000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986225,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":310990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.014,"15min":1.23},"min":{"1min":0.32000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.76600000000000001,"5min":15.547000000000001,"15min":15.547000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.58899999999999997,"15min":0.61599999999999999},"min":{"1min":0.34000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.79700000000000004,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61899999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55200000000000005,"15min":0.57199999999999995},"min":{"1min":0.32000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.72799999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58699999999999997,"15min":0.59699999999999998},"min":{"1min":0.38900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.85299999999999998,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.63900000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.59599999999999997},"min":{"1min":0.42499999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.755,"5min":0.873,"15min":0.873},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.59599999999999997,"15min":0.61199999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.78000000000000003,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.56799999999999995}]},{"osd":3,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.58499999999999996,"15min":0.59199999999999997},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79200000000000004,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58899999999999997,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78200000000000003,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66000000000000003}]},{"osd":5,"last update":"Fri Apr 26 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.45800000000000002,"15min":0.442},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.51900000000000002,"15min":0.501},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.41399999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:39:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.51200000000000001,"15min":0.501},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.46600000000000003}]},{"osd":7,"last update":"Fri Apr 26 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.55700000000000005,"15min":0.54900000000000004},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.53200000000000003,"15min":0.51800000000000002},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.51900000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724587,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":712704,"data_stored":309609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71599999999999997,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.56899999999999995,"15min":0.57899999999999996},"min":{"1min":0.37,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.67300000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.57699999999999996}]},{"osd":1,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56599999999999995,"15min":0.57599999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70199999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.58199999999999996,"15min":0.58899999999999997},"min":{"1min":0.36899999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72999999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.49299999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.67800000000000005,"15min":0.61799999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.268000000000001,"5min":11.268000000000001,"15min":11.268000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72599999999999998,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.55700000000000005}]},{"osd":3,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.67200000000000004,"15min":0.62},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":11.24,"5min":11.24,"15min":11.24},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.47099999999999997,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.81200000000000006,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.66000000000000003}]},{"osd":4,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.47799999999999998,"15min":0.46000000000000002},"min":{"1min":0.36799999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.44},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.47999999999999998,"15min":0.45200000000000001},"min":{"1min":0.36899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77500000000000002,"5min":1.244,"15min":1.244},"last":0.34699999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:39:46 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.505,"15min":0.49299999999999999},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51500000000000001,"15min":0.51400000000000001},"min":{"1min":0.30099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.38400000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.55200000000000005},"min":{"1min":0.40799999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78600000000000003,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.42599999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92600000000000005,"5min":1.173,"15min":1.173},"last":0.46200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:40:50.802 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T04:40:51.071 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:52.169 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:52 smithi038 ceph-mon[26454]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:52.169 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:52 smithi038 ceph-mon[26454]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:52.170 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:52 smithi038 ceph-mon[36743]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:52.170 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:52 smithi038 ceph-mon[36743]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:52 smithi055 ceph-mon[30328]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:52 smithi055 ceph-mon[30328]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:52.909 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:40:52.910 INFO:teuthology.orchestra.run.smithi038.stderr:dumped all 2024-04-26T04:40:53.543 INFO:teuthology.orchestra.run.smithi038.stdout:{"pg_ready":true,"pg_map":{"version":79,"stamp":"2024-04-26T04:40:50.994720+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386248,"kb_used_data":7380,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574968,"statfs":{"total":767960285184,"available":765516767232,"internally_reserved":0,"allocated":7557120,"data_stored":4252918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.911374"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286930+0000","last_change":"2024-04-26T04:39:56.321580+0000","last_active":"2024-04-26T04:40:39.286930+0000","last_peered":"2024-04-26T04:40:39.286930+0000","last_clean":"2024-04-26T04:40:39.286930+0000","last_became_active":"2024-04-26T04:39:56.321278+0000","last_became_peered":"2024-04-26T04:39:56.321278+0000","last_unstale":"2024-04-26T04:40:39.286930+0000","last_undegraded":"2024-04-26T04:40:39.286930+0000","last_fullsized":"2024-04-26T04:40:39.286930+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:43:21.733155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286763+0000","last_change":"2024-04-26T04:39:56.293313+0000","last_active":"2024-04-26T04:40:39.286763+0000","last_peered":"2024-04-26T04:40:39.286763+0000","last_clean":"2024-04-26T04:40:39.286763+0000","last_became_active":"2024-04-26T04:39:56.292965+0000","last_became_peered":"2024-04-26T04:39:56.292965+0000","last_unstale":"2024-04-26T04:40:39.286763+0000","last_undegraded":"2024-04-26T04:40:39.286763+0000","last_fullsized":"2024-04-26T04:40:39.286763+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:32:33.977936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056498+0000","last_change":"2024-04-26T04:39:56.314940+0000","last_active":"2024-04-26T04:40:39.056498+0000","last_peered":"2024-04-26T04:40:39.056498+0000","last_clean":"2024-04-26T04:40:39.056498+0000","last_became_active":"2024-04-26T04:39:56.314651+0000","last_became_peered":"2024-04-26T04:39:56.314651+0000","last_unstale":"2024-04-26T04:40:39.056498+0000","last_undegraded":"2024-04-26T04:40:39.056498+0000","last_fullsized":"2024-04-26T04:40:39.056498+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:07:45.865637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057141+0000","last_change":"2024-04-26T04:39:56.291555+0000","last_active":"2024-04-26T04:40:39.057141+0000","last_peered":"2024-04-26T04:40:39.057141+0000","last_clean":"2024-04-26T04:40:39.057141+0000","last_became_active":"2024-04-26T04:39:56.291266+0000","last_became_peered":"2024-04-26T04:39:56.291266+0000","last_unstale":"2024-04-26T04:40:39.057141+0000","last_undegraded":"2024-04-26T04:40:39.057141+0000","last_fullsized":"2024-04-26T04:40:39.057141+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:50:56.830886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287185+0000","last_change":"2024-04-26T04:39:56.314624+0000","last_active":"2024-04-26T04:40:39.287185+0000","last_peered":"2024-04-26T04:40:39.287185+0000","last_clean":"2024-04-26T04:40:39.287185+0000","last_became_active":"2024-04-26T04:39:56.314332+0000","last_became_peered":"2024-04-26T04:39:56.314332+0000","last_unstale":"2024-04-26T04:40:39.287185+0000","last_undegraded":"2024-04-26T04:40:39.287185+0000","last_fullsized":"2024-04-26T04:40:39.287185+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:35:09.627440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053230+0000","last_change":"2024-04-26T04:39:56.299459+0000","last_active":"2024-04-26T04:40:39.053230+0000","last_peered":"2024-04-26T04:40:39.053230+0000","last_clean":"2024-04-26T04:40:39.053230+0000","last_became_active":"2024-04-26T04:39:56.299180+0000","last_became_peered":"2024-04-26T04:39:56.299180+0000","last_unstale":"2024-04-26T04:40:39.053230+0000","last_undegraded":"2024-04-26T04:40:39.053230+0000","last_fullsized":"2024-04-26T04:40:39.053230+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:51:01.512924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287301+0000","last_change":"2024-04-26T04:39:56.318500+0000","last_active":"2024-04-26T04:40:39.287301+0000","last_peered":"2024-04-26T04:40:39.287301+0000","last_clean":"2024-04-26T04:40:39.287301+0000","last_became_active":"2024-04-26T04:39:56.318211+0000","last_became_peered":"2024-04-26T04:39:56.318211+0000","last_unstale":"2024-04-26T04:40:39.287301+0000","last_undegraded":"2024-04-26T04:40:39.287301+0000","last_fullsized":"2024-04-26T04:40:39.287301+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:53:04.341594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053236+0000","last_change":"2024-04-26T04:39:56.297710+0000","last_active":"2024-04-26T04:40:39.053236+0000","last_peered":"2024-04-26T04:40:39.053236+0000","last_clean":"2024-04-26T04:40:39.053236+0000","last_became_active":"2024-04-26T04:39:56.297435+0000","last_became_peered":"2024-04-26T04:39:56.297435+0000","last_unstale":"2024-04-26T04:40:39.053236+0000","last_undegraded":"2024-04-26T04:40:39.053236+0000","last_fullsized":"2024-04-26T04:40:39.053236+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:37:34.665357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:40:40.088755+0000","last_change":"2024-04-26T04:40:40.088755+0000","last_active":"2024-04-26T04:40:40.088755+0000","last_peered":"2024-04-26T04:40:40.088755+0000","last_clean":"2024-04-26T04:40:40.088755+0000","last_became_active":"2024-04-26T04:40:40.088473+0000","last_became_peered":"2024-04-26T04:40:40.088473+0000","last_unstale":"2024-04-26T04:40:40.088755+0000","last_undegraded":"2024-04-26T04:40:40.088755+0000","last_fullsized":"2024-04-26T04:40:40.088755+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:41:44.433736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286618+0000","last_change":"2024-04-26T04:39:56.297829+0000","last_active":"2024-04-26T04:40:39.286618+0000","last_peered":"2024-04-26T04:40:39.286618+0000","last_clean":"2024-04-26T04:40:39.286618+0000","last_became_active":"2024-04-26T04:39:56.297539+0000","last_became_peered":"2024-04-26T04:39:56.297539+0000","last_unstale":"2024-04-26T04:40:39.286618+0000","last_undegraded":"2024-04-26T04:40:39.286618+0000","last_fullsized":"2024-04-26T04:40:39.286618+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:20:28.665493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287124+0000","last_change":"2024-04-26T04:39:56.315667+0000","last_active":"2024-04-26T04:40:39.287124+0000","last_peered":"2024-04-26T04:40:39.287124+0000","last_clean":"2024-04-26T04:40:39.287124+0000","last_became_active":"2024-04-26T04:39:56.315353+0000","last_became_peered":"2024-04-26T04:39:56.315353+0000","last_unstale":"2024-04-26T04:40:39.287124+0000","last_undegraded":"2024-04-26T04:40:39.287124+0000","last_fullsized":"2024-04-26T04:40:39.287124+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:45:00.659197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286647+0000","last_change":"2024-04-26T04:39:56.296410+0000","last_active":"2024-04-26T04:40:39.286647+0000","last_peered":"2024-04-26T04:40:39.286647+0000","last_clean":"2024-04-26T04:40:39.286647+0000","last_became_active":"2024-04-26T04:39:56.296116+0000","last_became_peered":"2024-04-26T04:39:56.296116+0000","last_unstale":"2024-04-26T04:40:39.286647+0000","last_undegraded":"2024-04-26T04:40:39.286647+0000","last_fullsized":"2024-04-26T04:40:39.286647+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:35:53.479201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056093+0000","last_change":"2024-04-26T04:38:54.064213+0000","last_active":"2024-04-26T04:40:39.056093+0000","last_peered":"2024-04-26T04:40:39.056093+0000","last_clean":"2024-04-26T04:40:39.056093+0000","last_became_active":"2024-04-26T04:38:54.063797+0000","last_became_peered":"2024-04-26T04:38:54.063797+0000","last_unstale":"2024-04-26T04:40:39.056093+0000","last_undegraded":"2024-04-26T04:40:39.056093+0000","last_fullsized":"2024-04-26T04:40:39.056093+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:21.370670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056073+0000","last_change":"2024-04-26T04:39:56.924091+0000","last_active":"2024-04-26T04:40:39.056073+0000","last_peered":"2024-04-26T04:40:39.056073+0000","last_clean":"2024-04-26T04:40:39.056073+0000","last_became_active":"2024-04-26T04:39:56.923807+0000","last_became_peered":"2024-04-26T04:39:56.923807+0000","last_unstale":"2024-04-26T04:40:39.056073+0000","last_undegraded":"2024-04-26T04:40:39.056073+0000","last_fullsized":"2024-04-26T04:40:39.056073+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:06:01.401463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059986+0000","last_change":"2024-04-26T04:38:54.530923+0000","last_active":"2024-04-26T04:40:39.059986+0000","last_peered":"2024-04-26T04:40:39.059986+0000","last_clean":"2024-04-26T04:40:39.059986+0000","last_became_active":"2024-04-26T04:38:54.530540+0000","last_became_peered":"2024-04-26T04:38:54.530540+0000","last_unstale":"2024-04-26T04:40:39.059986+0000","last_undegraded":"2024-04-26T04:40:39.059986+0000","last_fullsized":"2024-04-26T04:40:39.059986+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:28:56.675606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057127+0000","last_change":"2024-04-26T04:39:56.299210+0000","last_active":"2024-04-26T04:40:39.057127+0000","last_peered":"2024-04-26T04:40:39.057127+0000","last_clean":"2024-04-26T04:40:39.057127+0000","last_became_active":"2024-04-26T04:39:56.298905+0000","last_became_peered":"2024-04-26T04:39:56.298905+0000","last_unstale":"2024-04-26T04:40:39.057127+0000","last_undegraded":"2024-04-26T04:40:39.057127+0000","last_fullsized":"2024-04-26T04:40:39.057127+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:38:35.254187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286555+0000","last_change":"2024-04-26T04:39:56.923802+0000","last_active":"2024-04-26T04:40:39.286555+0000","last_peered":"2024-04-26T04:40:39.286555+0000","last_clean":"2024-04-26T04:40:39.286555+0000","last_became_active":"2024-04-26T04:39:56.923501+0000","last_became_peered":"2024-04-26T04:39:56.923501+0000","last_unstale":"2024-04-26T04:40:39.286555+0000","last_undegraded":"2024-04-26T04:40:39.286555+0000","last_fullsized":"2024-04-26T04:40:39.286555+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:00:46.110996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059431+0000","last_change":"2024-04-26T04:39:56.293821+0000","last_active":"2024-04-26T04:40:39.059431+0000","last_peered":"2024-04-26T04:40:39.059431+0000","last_clean":"2024-04-26T04:40:39.059431+0000","last_became_active":"2024-04-26T04:39:56.292175+0000","last_became_peered":"2024-04-26T04:39:56.292175+0000","last_unstale":"2024-04-26T04:40:39.059431+0000","last_undegraded":"2024-04-26T04:40:39.059431+0000","last_fullsized":"2024-04-26T04:40:39.059431+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:40:55.049762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057715+0000","last_change":"2024-04-26T04:38:54.530974+0000","last_active":"2024-04-26T04:40:39.057715+0000","last_peered":"2024-04-26T04:40:39.057715+0000","last_clean":"2024-04-26T04:40:39.057715+0000","last_became_active":"2024-04-26T04:38:54.530538+0000","last_became_peered":"2024-04-26T04:38:54.530538+0000","last_unstale":"2024-04-26T04:40:39.057715+0000","last_undegraded":"2024-04-26T04:40:39.057715+0000","last_fullsized":"2024-04-26T04:40:39.057715+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:40:04.608877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286690+0000","last_change":"2024-04-26T04:39:56.318781+0000","last_active":"2024-04-26T04:40:39.286690+0000","last_peered":"2024-04-26T04:40:39.286690+0000","last_clean":"2024-04-26T04:40:39.286690+0000","last_became_active":"2024-04-26T04:39:56.318414+0000","last_became_peered":"2024-04-26T04:39:56.318414+0000","last_unstale":"2024-04-26T04:40:39.286690+0000","last_undegraded":"2024-04-26T04:40:39.286690+0000","last_fullsized":"2024-04-26T04:40:39.286690+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:56:28.545804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286436+0000","last_change":"2024-04-26T04:39:56.298765+0000","last_active":"2024-04-26T04:40:39.286436+0000","last_peered":"2024-04-26T04:40:39.286436+0000","last_clean":"2024-04-26T04:40:39.286436+0000","last_became_active":"2024-04-26T04:39:56.298496+0000","last_became_peered":"2024-04-26T04:39:56.298496+0000","last_unstale":"2024-04-26T04:40:39.286436+0000","last_undegraded":"2024-04-26T04:40:39.286436+0000","last_fullsized":"2024-04-26T04:40:39.286436+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:34:31.926911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057383+0000","last_change":"2024-04-26T04:38:54.065770+0000","last_active":"2024-04-26T04:40:39.057383+0000","last_peered":"2024-04-26T04:40:39.057383+0000","last_clean":"2024-04-26T04:40:39.057383+0000","last_became_active":"2024-04-26T04:38:54.065409+0000","last_became_peered":"2024-04-26T04:38:54.065409+0000","last_unstale":"2024-04-26T04:40:39.057383+0000","last_undegraded":"2024-04-26T04:40:39.057383+0000","last_fullsized":"2024-04-26T04:40:39.057383+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:41:56.230650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.055488+0000","last_change":"2024-04-26T04:39:56.923636+0000","last_active":"2024-04-26T04:40:39.055488+0000","last_peered":"2024-04-26T04:40:39.055488+0000","last_clean":"2024-04-26T04:40:39.055488+0000","last_became_active":"2024-04-26T04:39:56.923324+0000","last_became_peered":"2024-04-26T04:39:56.923324+0000","last_unstale":"2024-04-26T04:40:39.055488+0000","last_undegraded":"2024-04-26T04:40:39.055488+0000","last_fullsized":"2024-04-26T04:40:39.055488+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:05:54.903060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056358+0000","last_change":"2024-04-26T04:39:56.291091+0000","last_active":"2024-04-26T04:40:39.056358+0000","last_peered":"2024-04-26T04:40:39.056358+0000","last_clean":"2024-04-26T04:40:39.056358+0000","last_became_active":"2024-04-26T04:39:56.290764+0000","last_became_peered":"2024-04-26T04:39:56.290764+0000","last_unstale":"2024-04-26T04:40:39.056358+0000","last_undegraded":"2024-04-26T04:40:39.056358+0000","last_fullsized":"2024-04-26T04:40:39.056358+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:40:14.466242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057304+0000","last_change":"2024-04-26T04:38:11.717965+0000","last_active":"2024-04-26T04:40:39.057304+0000","last_peered":"2024-04-26T04:40:39.057304+0000","last_clean":"2024-04-26T04:40:39.057304+0000","last_became_active":"2024-04-26T04:38:11.717490+0000","last_became_peered":"2024-04-26T04:38:11.717490+0000","last_unstale":"2024-04-26T04:40:39.057304+0000","last_undegraded":"2024-04-26T04:40:39.057304+0000","last_fullsized":"2024-04-26T04:40:39.057304+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_clean_scrub_stamp":"2024-04-26T04:36:10.749042+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:18:38.216134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053025+0000","last_change":"2024-04-26T04:38:54.063499+0000","last_active":"2024-04-26T04:40:39.053025+0000","last_peered":"2024-04-26T04:40:39.053025+0000","last_clean":"2024-04-26T04:40:39.053025+0000","last_became_active":"2024-04-26T04:38:54.063035+0000","last_became_peered":"2024-04-26T04:38:54.063035+0000","last_unstale":"2024-04-26T04:40:39.053025+0000","last_undegraded":"2024-04-26T04:40:39.053025+0000","last_fullsized":"2024-04-26T04:40:39.053025+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:20.279953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060170+0000","last_change":"2024-04-26T04:39:56.323989+0000","last_active":"2024-04-26T04:40:39.060170+0000","last_peered":"2024-04-26T04:40:39.060170+0000","last_clean":"2024-04-26T04:40:39.060170+0000","last_became_active":"2024-04-26T04:39:56.322916+0000","last_became_peered":"2024-04-26T04:39:56.322916+0000","last_unstale":"2024-04-26T04:40:39.060170+0000","last_undegraded":"2024-04-26T04:40:39.060170+0000","last_fullsized":"2024-04-26T04:40:39.060170+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:44:47.533970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059343+0000","last_change":"2024-04-26T04:39:56.293415+0000","last_active":"2024-04-26T04:40:39.059343+0000","last_peered":"2024-04-26T04:40:39.059343+0000","last_clean":"2024-04-26T04:40:39.059343+0000","last_became_active":"2024-04-26T04:39:56.291575+0000","last_became_peered":"2024-04-26T04:39:56.291575+0000","last_unstale":"2024-04-26T04:40:39.059343+0000","last_undegraded":"2024-04-26T04:40:39.059343+0000","last_fullsized":"2024-04-26T04:40:39.059343+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:51:20.825043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056244+0000","last_change":"2024-04-26T04:38:54.066302+0000","last_active":"2024-04-26T04:40:39.056244+0000","last_peered":"2024-04-26T04:40:39.056244+0000","last_clean":"2024-04-26T04:40:39.056244+0000","last_became_active":"2024-04-26T04:38:54.065974+0000","last_became_peered":"2024-04-26T04:38:54.065974+0000","last_unstale":"2024-04-26T04:40:39.056244+0000","last_undegraded":"2024-04-26T04:40:39.056244+0000","last_fullsized":"2024-04-26T04:40:39.056244+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:27:22.809404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059347+0000","last_change":"2024-04-26T04:39:56.324186+0000","last_active":"2024-04-26T04:40:39.059347+0000","last_peered":"2024-04-26T04:40:39.059347+0000","last_clean":"2024-04-26T04:40:39.059347+0000","last_became_active":"2024-04-26T04:39:56.323936+0000","last_became_peered":"2024-04-26T04:39:56.323936+0000","last_unstale":"2024-04-26T04:40:39.059347+0000","last_undegraded":"2024-04-26T04:40:39.059347+0000","last_fullsized":"2024-04-26T04:40:39.059347+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:56:56.391281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287242+0000","last_change":"2024-04-26T04:39:56.292521+0000","last_active":"2024-04-26T04:40:39.287242+0000","last_peered":"2024-04-26T04:40:39.287242+0000","last_clean":"2024-04-26T04:40:39.287242+0000","last_became_active":"2024-04-26T04:39:56.292247+0000","last_became_peered":"2024-04-26T04:39:56.292247+0000","last_unstale":"2024-04-26T04:40:39.287242+0000","last_undegraded":"2024-04-26T04:40:39.287242+0000","last_fullsized":"2024-04-26T04:40:39.287242+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T04:47:20.168803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.052978+0000","last_change":"2024-04-26T04:38:54.064959+0000","last_active":"2024-04-26T04:40:39.052978+0000","last_peered":"2024-04-26T04:40:39.052978+0000","last_clean":"2024-04-26T04:40:39.052978+0000","last_became_active":"2024-04-26T04:38:54.064530+0000","last_became_peered":"2024-04-26T04:38:54.064530+0000","last_unstale":"2024-04-26T04:40:39.052978+0000","last_undegraded":"2024-04-26T04:40:39.052978+0000","last_fullsized":"2024-04-26T04:40:39.052978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:20.559351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060106+0000","last_change":"2024-04-26T04:39:56.322825+0000","last_active":"2024-04-26T04:40:39.060106+0000","last_peered":"2024-04-26T04:40:39.060106+0000","last_clean":"2024-04-26T04:40:39.060106+0000","last_became_active":"2024-04-26T04:39:56.322517+0000","last_became_peered":"2024-04-26T04:39:56.322517+0000","last_unstale":"2024-04-26T04:40:39.060106+0000","last_undegraded":"2024-04-26T04:40:39.060106+0000","last_fullsized":"2024-04-26T04:40:39.060106+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:50:08.121628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053199+0000","last_change":"2024-04-26T04:39:56.292684+0000","last_active":"2024-04-26T04:40:39.053199+0000","last_peered":"2024-04-26T04:40:39.053199+0000","last_clean":"2024-04-26T04:40:39.053199+0000","last_became_active":"2024-04-26T04:39:56.291835+0000","last_became_peered":"2024-04-26T04:39:56.291835+0000","last_unstale":"2024-04-26T04:40:39.053199+0000","last_undegraded":"2024-04-26T04:40:39.053199+0000","last_fullsized":"2024-04-26T04:40:39.053199+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:27:12.540494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059265+0000","last_change":"2024-04-26T04:38:54.065701+0000","last_active":"2024-04-26T04:40:39.059265+0000","last_peered":"2024-04-26T04:40:39.059265+0000","last_clean":"2024-04-26T04:40:39.059265+0000","last_became_active":"2024-04-26T04:38:54.065370+0000","last_became_peered":"2024-04-26T04:38:54.065370+0000","last_unstale":"2024-04-26T04:40:39.059265+0000","last_undegraded":"2024-04-26T04:40:39.059265+0000","last_fullsized":"2024-04-26T04:40:39.059265+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:50:38.001692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060257+0000","last_change":"2024-04-26T04:39:56.324150+0000","last_active":"2024-04-26T04:40:39.060257+0000","last_peered":"2024-04-26T04:40:39.060257+0000","last_clean":"2024-04-26T04:40:39.060257+0000","last_became_active":"2024-04-26T04:39:56.323725+0000","last_became_peered":"2024-04-26T04:39:56.323725+0000","last_unstale":"2024-04-26T04:40:39.060257+0000","last_undegraded":"2024-04-26T04:40:39.060257+0000","last_fullsized":"2024-04-26T04:40:39.060257+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:33:11.245827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056183+0000","last_change":"2024-04-26T04:39:56.301577+0000","last_active":"2024-04-26T04:40:39.056183+0000","last_peered":"2024-04-26T04:40:39.056183+0000","last_clean":"2024-04-26T04:40:39.056183+0000","last_became_active":"2024-04-26T04:39:56.301060+0000","last_became_peered":"2024-04-26T04:39:56.301060+0000","last_unstale":"2024-04-26T04:40:39.056183+0000","last_undegraded":"2024-04-26T04:40:39.056183+0000","last_fullsized":"2024-04-26T04:40:39.056183+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:31:39.931897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287614+0000","last_change":"2024-04-26T04:39:56.316678+0000","last_active":"2024-04-26T04:40:39.287614+0000","last_peered":"2024-04-26T04:40:39.287614+0000","last_clean":"2024-04-26T04:40:39.287614+0000","last_became_active":"2024-04-26T04:39:56.316374+0000","last_became_peered":"2024-04-26T04:39:56.316374+0000","last_unstale":"2024-04-26T04:40:39.287614+0000","last_undegraded":"2024-04-26T04:40:39.287614+0000","last_fullsized":"2024-04-26T04:40:39.287614+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:50:38.289489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287182+0000","last_change":"2024-04-26T04:39:56.298990+0000","last_active":"2024-04-26T04:40:39.287182+0000","last_peered":"2024-04-26T04:40:39.287182+0000","last_clean":"2024-04-26T04:40:39.287182+0000","last_became_active":"2024-04-26T04:39:56.298660+0000","last_became_peered":"2024-04-26T04:39:56.298660+0000","last_unstale":"2024-04-26T04:40:39.287182+0000","last_undegraded":"2024-04-26T04:40:39.287182+0000","last_fullsized":"2024-04-26T04:40:39.287182+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:48:46.486720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287561+0000","last_change":"2024-04-26T04:39:56.318571+0000","last_active":"2024-04-26T04:40:39.287561+0000","last_peered":"2024-04-26T04:40:39.287561+0000","last_clean":"2024-04-26T04:40:39.287561+0000","last_became_active":"2024-04-26T04:39:56.318273+0000","last_became_peered":"2024-04-26T04:39:56.318273+0000","last_unstale":"2024-04-26T04:40:39.287561+0000","last_undegraded":"2024-04-26T04:40:39.287561+0000","last_fullsized":"2024-04-26T04:40:39.287561+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:54:58.284911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.055585+0000","last_change":"2024-04-26T04:39:56.294993+0000","last_active":"2024-04-26T04:40:39.055585+0000","last_peered":"2024-04-26T04:40:39.055585+0000","last_clean":"2024-04-26T04:40:39.055585+0000","last_became_active":"2024-04-26T04:39:56.294723+0000","last_became_peered":"2024-04-26T04:39:56.294723+0000","last_unstale":"2024-04-26T04:40:39.055585+0000","last_undegraded":"2024-04-26T04:40:39.055585+0000","last_fullsized":"2024-04-26T04:40:39.055585+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:01:25.922226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287314+0000","last_change":"2024-04-26T04:39:56.315908+0000","last_active":"2024-04-26T04:40:39.287314+0000","last_peered":"2024-04-26T04:40:39.287314+0000","last_clean":"2024-04-26T04:40:39.287314+0000","last_became_active":"2024-04-26T04:39:56.315616+0000","last_became_peered":"2024-04-26T04:39:56.315616+0000","last_unstale":"2024-04-26T04:40:39.287314+0000","last_undegraded":"2024-04-26T04:40:39.287314+0000","last_fullsized":"2024-04-26T04:40:39.287314+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:23:47.585151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056864+0000","last_change":"2024-04-26T04:39:56.298264+0000","last_active":"2024-04-26T04:40:39.056864+0000","last_peered":"2024-04-26T04:40:39.056864+0000","last_clean":"2024-04-26T04:40:39.056864+0000","last_became_active":"2024-04-26T04:39:56.297945+0000","last_became_peered":"2024-04-26T04:39:56.297945+0000","last_unstale":"2024-04-26T04:40:39.056864+0000","last_undegraded":"2024-04-26T04:40:39.056864+0000","last_fullsized":"2024-04-26T04:40:39.056864+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:15:20.706729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286467+0000","last_change":"2024-04-26T04:39:56.319985+0000","last_active":"2024-04-26T04:40:39.286467+0000","last_peered":"2024-04-26T04:40:39.286467+0000","last_clean":"2024-04-26T04:40:39.286467+0000","last_became_active":"2024-04-26T04:39:56.319677+0000","last_became_peered":"2024-04-26T04:39:56.319677+0000","last_unstale":"2024-04-26T04:40:39.286467+0000","last_undegraded":"2024-04-26T04:40:39.286467+0000","last_fullsized":"2024-04-26T04:40:39.286467+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:06:56.732447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053141+0000","last_change":"2024-04-26T04:39:56.295923+0000","last_active":"2024-04-26T04:40:39.053141+0000","last_peered":"2024-04-26T04:40:39.053141+0000","last_clean":"2024-04-26T04:40:39.053141+0000","last_became_active":"2024-04-26T04:39:56.295537+0000","last_became_peered":"2024-04-26T04:39:56.295537+0000","last_unstale":"2024-04-26T04:40:39.053141+0000","last_undegraded":"2024-04-26T04:40:39.053141+0000","last_fullsized":"2024-04-26T04:40:39.053141+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:25:33.453364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:40:40.450338+0000","last_change":"2024-04-26T04:40:40.450338+0000","last_active":"2024-04-26T04:40:40.450338+0000","last_peered":"2024-04-26T04:40:40.450338+0000","last_clean":"2024-04-26T04:40:40.450338+0000","last_became_active":"2024-04-26T04:40:40.449961+0000","last_became_peered":"2024-04-26T04:40:40.449961+0000","last_unstale":"2024-04-26T04:40:40.450338+0000","last_undegraded":"2024-04-26T04:40:40.450338+0000","last_fullsized":"2024-04-26T04:40:40.450338+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:06:03.706318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057825+0000","last_change":"2024-04-26T04:39:56.300360+0000","last_active":"2024-04-26T04:40:39.057825+0000","last_peered":"2024-04-26T04:40:39.057825+0000","last_clean":"2024-04-26T04:40:39.057825+0000","last_became_active":"2024-04-26T04:39:56.300067+0000","last_became_peered":"2024-04-26T04:39:56.300067+0000","last_unstale":"2024-04-26T04:40:39.057825+0000","last_undegraded":"2024-04-26T04:40:39.057825+0000","last_fullsized":"2024-04-26T04:40:39.057825+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:53:28.753150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287044+0000","last_change":"2024-04-26T04:39:56.318542+0000","last_active":"2024-04-26T04:40:39.287044+0000","last_peered":"2024-04-26T04:40:39.287044+0000","last_clean":"2024-04-26T04:40:39.287044+0000","last_became_active":"2024-04-26T04:39:56.318294+0000","last_became_peered":"2024-04-26T04:39:56.318294+0000","last_unstale":"2024-04-26T04:40:39.287044+0000","last_undegraded":"2024-04-26T04:40:39.287044+0000","last_fullsized":"2024-04-26T04:40:39.287044+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:54:41.940567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057699+0000","last_change":"2024-04-26T04:39:56.304876+0000","last_active":"2024-04-26T04:40:39.057699+0000","last_peered":"2024-04-26T04:40:39.057699+0000","last_clean":"2024-04-26T04:40:39.057699+0000","last_became_active":"2024-04-26T04:39:56.304597+0000","last_became_peered":"2024-04-26T04:39:56.304597+0000","last_unstale":"2024-04-26T04:40:39.057699+0000","last_undegraded":"2024-04-26T04:40:39.057699+0000","last_fullsized":"2024-04-26T04:40:39.057699+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:16:18.446961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.052921+0000","last_change":"2024-04-26T04:39:56.925632+0000","last_active":"2024-04-26T04:40:39.052921+0000","last_peered":"2024-04-26T04:40:39.052921+0000","last_clean":"2024-04-26T04:40:39.052921+0000","last_became_active":"2024-04-26T04:39:56.925327+0000","last_became_peered":"2024-04-26T04:39:56.925327+0000","last_unstale":"2024-04-26T04:40:39.052921+0000","last_undegraded":"2024-04-26T04:40:39.052921+0000","last_fullsized":"2024-04-26T04:40:39.052921+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:29:59.594965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.058017+0000","last_change":"2024-04-26T04:39:56.304531+0000","last_active":"2024-04-26T04:40:39.058017+0000","last_peered":"2024-04-26T04:40:39.058017+0000","last_clean":"2024-04-26T04:40:39.058017+0000","last_became_active":"2024-04-26T04:39:56.304209+0000","last_became_peered":"2024-04-26T04:39:56.304209+0000","last_unstale":"2024-04-26T04:40:39.058017+0000","last_undegraded":"2024-04-26T04:40:39.058017+0000","last_fullsized":"2024-04-26T04:40:39.058017+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:01:36.403239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056872+0000","last_change":"2024-04-26T04:39:56.925049+0000","last_active":"2024-04-26T04:40:39.056872+0000","last_peered":"2024-04-26T04:40:39.056872+0000","last_clean":"2024-04-26T04:40:39.056872+0000","last_became_active":"2024-04-26T04:39:56.924725+0000","last_became_peered":"2024-04-26T04:39:56.924725+0000","last_unstale":"2024-04-26T04:40:39.056872+0000","last_undegraded":"2024-04-26T04:40:39.056872+0000","last_fullsized":"2024-04-26T04:40:39.056872+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:46:34.564332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":21,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.055359+0000","last_change":"2024-04-26T04:39:56.293952+0000","last_active":"2024-04-26T04:40:39.055359+0000","last_peered":"2024-04-26T04:40:39.055359+0000","last_clean":"2024-04-26T04:40:39.055359+0000","last_became_active":"2024-04-26T04:39:56.293651+0000","last_became_peered":"2024-04-26T04:39:56.293651+0000","last_unstale":"2024-04-26T04:40:39.055359+0000","last_undegraded":"2024-04-26T04:40:39.055359+0000","last_fullsized":"2024-04-26T04:40:39.055359+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:13:19.299051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286549+0000","last_change":"2024-04-26T04:39:56.923289+0000","last_active":"2024-04-26T04:40:39.286549+0000","last_peered":"2024-04-26T04:40:39.286549+0000","last_clean":"2024-04-26T04:40:39.286549+0000","last_became_active":"2024-04-26T04:39:56.922986+0000","last_became_peered":"2024-04-26T04:39:56.922986+0000","last_unstale":"2024-04-26T04:40:39.286549+0000","last_undegraded":"2024-04-26T04:40:39.286549+0000","last_fullsized":"2024-04-26T04:40:39.286549+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:36:03.904766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057317+0000","last_change":"2024-04-26T04:39:56.302393+0000","last_active":"2024-04-26T04:40:39.057317+0000","last_peered":"2024-04-26T04:40:39.057317+0000","last_clean":"2024-04-26T04:40:39.057317+0000","last_became_active":"2024-04-26T04:39:56.302116+0000","last_became_peered":"2024-04-26T04:39:56.302116+0000","last_unstale":"2024-04-26T04:40:39.057317+0000","last_undegraded":"2024-04-26T04:40:39.057317+0000","last_fullsized":"2024-04-26T04:40:39.057317+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:28:13.475580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053118+0000","last_change":"2024-04-26T04:39:56.924653+0000","last_active":"2024-04-26T04:40:39.053118+0000","last_peered":"2024-04-26T04:40:39.053118+0000","last_clean":"2024-04-26T04:40:39.053118+0000","last_became_active":"2024-04-26T04:39:56.924283+0000","last_became_peered":"2024-04-26T04:39:56.924283+0000","last_unstale":"2024-04-26T04:40:39.053118+0000","last_undegraded":"2024-04-26T04:40:39.053118+0000","last_fullsized":"2024-04-26T04:40:39.053118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:07:19.121146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286438+0000","last_change":"2024-04-26T04:39:56.300183+0000","last_active":"2024-04-26T04:40:39.286438+0000","last_peered":"2024-04-26T04:40:39.286438+0000","last_clean":"2024-04-26T04:40:39.286438+0000","last_became_active":"2024-04-26T04:39:56.299703+0000","last_became_peered":"2024-04-26T04:39:56.299703+0000","last_unstale":"2024-04-26T04:40:39.286438+0000","last_undegraded":"2024-04-26T04:40:39.286438+0000","last_fullsized":"2024-04-26T04:40:39.286438+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:33:28.080217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057095+0000","last_change":"2024-04-26T04:39:56.922951+0000","last_active":"2024-04-26T04:40:39.057095+0000","last_peered":"2024-04-26T04:40:39.057095+0000","last_clean":"2024-04-26T04:40:39.057095+0000","last_became_active":"2024-04-26T04:39:56.922592+0000","last_became_peered":"2024-04-26T04:39:56.922592+0000","last_unstale":"2024-04-26T04:40:39.057095+0000","last_undegraded":"2024-04-26T04:40:39.057095+0000","last_fullsized":"2024-04-26T04:40:39.057095+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:07:26.648362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":15,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.057952+0000","last_change":"2024-04-26T04:39:56.304345+0000","last_active":"2024-04-26T04:40:39.057952+0000","last_peered":"2024-04-26T04:40:39.057952+0000","last_clean":"2024-04-26T04:40:39.057952+0000","last_became_active":"2024-04-26T04:39:56.304040+0000","last_became_peered":"2024-04-26T04:39:56.304040+0000","last_unstale":"2024-04-26T04:40:39.057952+0000","last_undegraded":"2024-04-26T04:40:39.057952+0000","last_fullsized":"2024-04-26T04:40:39.057952+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:37:54.252912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287204+0000","last_change":"2024-04-26T04:39:56.318742+0000","last_active":"2024-04-26T04:40:39.287204+0000","last_peered":"2024-04-26T04:40:39.287204+0000","last_clean":"2024-04-26T04:40:39.287204+0000","last_became_active":"2024-04-26T04:39:56.318441+0000","last_became_peered":"2024-04-26T04:39:56.318441+0000","last_unstale":"2024-04-26T04:40:39.287204+0000","last_undegraded":"2024-04-26T04:40:39.287204+0000","last_fullsized":"2024-04-26T04:40:39.287204+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:55:08.709896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287391+0000","last_change":"2024-04-26T04:39:56.300576+0000","last_active":"2024-04-26T04:40:39.287391+0000","last_peered":"2024-04-26T04:40:39.287391+0000","last_clean":"2024-04-26T04:40:39.287391+0000","last_became_active":"2024-04-26T04:39:56.299260+0000","last_became_peered":"2024-04-26T04:39:56.299260+0000","last_unstale":"2024-04-26T04:40:39.287391+0000","last_undegraded":"2024-04-26T04:40:39.287391+0000","last_fullsized":"2024-04-26T04:40:39.287391+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:26.586898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.060169+0000","last_change":"2024-04-26T04:39:56.323585+0000","last_active":"2024-04-26T04:40:39.060169+0000","last_peered":"2024-04-26T04:40:39.060169+0000","last_clean":"2024-04-26T04:40:39.060169+0000","last_became_active":"2024-04-26T04:39:56.323322+0000","last_became_peered":"2024-04-26T04:39:56.323322+0000","last_unstale":"2024-04-26T04:40:39.060169+0000","last_undegraded":"2024-04-26T04:40:39.060169+0000","last_fullsized":"2024-04-26T04:40:39.060169+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:08:11.656124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":16,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.058059+0000","last_change":"2024-04-26T04:39:56.304338+0000","last_active":"2024-04-26T04:40:39.058059+0000","last_peered":"2024-04-26T04:40:39.058059+0000","last_clean":"2024-04-26T04:40:39.058059+0000","last_became_active":"2024-04-26T04:39:56.304033+0000","last_became_peered":"2024-04-26T04:39:56.304033+0000","last_unstale":"2024-04-26T04:40:39.058059+0000","last_undegraded":"2024-04-26T04:40:39.058059+0000","last_fullsized":"2024-04-26T04:40:39.058059+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:11:57.154652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056308+0000","last_change":"2024-04-26T04:39:56.923328+0000","last_active":"2024-04-26T04:40:39.056308+0000","last_peered":"2024-04-26T04:40:39.056308+0000","last_clean":"2024-04-26T04:40:39.056308+0000","last_became_active":"2024-04-26T04:39:56.923032+0000","last_became_peered":"2024-04-26T04:39:56.923032+0000","last_unstale":"2024-04-26T04:40:39.056308+0000","last_undegraded":"2024-04-26T04:40:39.056308+0000","last_fullsized":"2024-04-26T04:40:39.056308+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:34:11.959235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.053296+0000","last_change":"2024-04-26T04:39:56.297527+0000","last_active":"2024-04-26T04:40:39.053296+0000","last_peered":"2024-04-26T04:40:39.053296+0000","last_clean":"2024-04-26T04:40:39.053296+0000","last_became_active":"2024-04-26T04:39:56.297235+0000","last_became_peered":"2024-04-26T04:39:56.297235+0000","last_unstale":"2024-04-26T04:40:39.053296+0000","last_undegraded":"2024-04-26T04:40:39.053296+0000","last_fullsized":"2024-04-26T04:40:39.053296+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:27:53.928583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056264+0000","last_change":"2024-04-26T04:39:56.321557+0000","last_active":"2024-04-26T04:40:39.056264+0000","last_peered":"2024-04-26T04:40:39.056264+0000","last_clean":"2024-04-26T04:40:39.056264+0000","last_became_active":"2024-04-26T04:39:56.321228+0000","last_became_peered":"2024-04-26T04:39:56.321228+0000","last_unstale":"2024-04-26T04:40:39.056264+0000","last_undegraded":"2024-04-26T04:40:39.056264+0000","last_fullsized":"2024-04-26T04:40:39.056264+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:18:27.526771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286710+0000","last_change":"2024-04-26T04:39:56.299978+0000","last_active":"2024-04-26T04:40:39.286710+0000","last_peered":"2024-04-26T04:40:39.286710+0000","last_clean":"2024-04-26T04:40:39.286710+0000","last_became_active":"2024-04-26T04:39:56.299603+0000","last_became_peered":"2024-04-26T04:39:56.299603+0000","last_unstale":"2024-04-26T04:40:39.286710+0000","last_undegraded":"2024-04-26T04:40:39.286710+0000","last_fullsized":"2024-04-26T04:40:39.286710+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:03:24.553191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-26T04:40:40.088501+0000","last_change":"2024-04-26T04:40:40.088501+0000","last_active":"2024-04-26T04:40:40.088501+0000","last_peered":"2024-04-26T04:40:40.088501+0000","last_clean":"2024-04-26T04:40:40.088501+0000","last_became_active":"2024-04-26T04:40:40.088060+0000","last_became_peered":"2024-04-26T04:40:40.088060+0000","last_unstale":"2024-04-26T04:40:40.088501+0000","last_undegraded":"2024-04-26T04:40:40.088501+0000","last_fullsized":"2024-04-26T04:40:40.088501+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:04:13.569242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056359+0000","last_change":"2024-04-26T04:39:56.292587+0000","last_active":"2024-04-26T04:40:39.056359+0000","last_peered":"2024-04-26T04:40:39.056359+0000","last_clean":"2024-04-26T04:40:39.056359+0000","last_became_active":"2024-04-26T04:39:56.292332+0000","last_became_peered":"2024-04-26T04:39:56.292332+0000","last_unstale":"2024-04-26T04:40:39.056359+0000","last_undegraded":"2024-04-26T04:40:39.056359+0000","last_fullsized":"2024-04-26T04:40:39.056359+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:19:26.474036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.286602+0000","last_change":"2024-04-26T04:39:56.923162+0000","last_active":"2024-04-26T04:40:39.286602+0000","last_peered":"2024-04-26T04:40:39.286602+0000","last_clean":"2024-04-26T04:40:39.286602+0000","last_became_active":"2024-04-26T04:39:56.922828+0000","last_became_peered":"2024-04-26T04:39:56.922828+0000","last_unstale":"2024-04-26T04:40:39.286602+0000","last_undegraded":"2024-04-26T04:40:39.286602+0000","last_fullsized":"2024-04-26T04:40:39.286602+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:20:11.342915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.059620+0000","last_change":"2024-04-26T04:39:56.294919+0000","last_active":"2024-04-26T04:40:39.059620+0000","last_peered":"2024-04-26T04:40:39.059620+0000","last_clean":"2024-04-26T04:40:39.059620+0000","last_became_active":"2024-04-26T04:39:56.294617+0000","last_became_peered":"2024-04-26T04:39:56.294617+0000","last_unstale":"2024-04-26T04:40:39.059620+0000","last_undegraded":"2024-04-26T04:40:39.059620+0000","last_fullsized":"2024-04-26T04:40:39.059620+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:22:16.623574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.056470+0000","last_change":"2024-04-26T04:39:56.320548+0000","last_active":"2024-04-26T04:40:39.056470+0000","last_peered":"2024-04-26T04:40:39.056470+0000","last_clean":"2024-04-26T04:40:39.056470+0000","last_became_active":"2024-04-26T04:39:56.320269+0000","last_became_peered":"2024-04-26T04:39:56.320269+0000","last_unstale":"2024-04-26T04:40:39.056470+0000","last_undegraded":"2024-04-26T04:40:39.056470+0000","last_fullsized":"2024-04-26T04:40:39.056470+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:17:07.659869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-26T04:40:39.287058+0000","last_change":"2024-04-26T04:39:56.293361+0000","last_active":"2024-04-26T04:40:39.287058+0000","last_peered":"2024-04-26T04:40:39.287058+0000","last_clean":"2024-04-26T04:40:39.287058+0000","last_became_active":"2024-04-26T04:39:56.293064+0000","last_became_peered":"2024-04-26T04:39:56.293064+0000","last_unstale":"2024-04-26T04:40:39.287058+0000","last_undegraded":"2024-04-26T04:40:39.287058+0000","last_fullsized":"2024-04-26T04:40:39.287058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:57:47.052446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201315,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":901155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.40899999999999997,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.45500000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.53100000000000003,"15min":0.52300000000000002},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.66100000000000003}]},{"osd":2,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.53300000000000003,"15min":0.52700000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.65300000000000002,"15min":0.60299999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.9450000000000003,"5min":7.9450000000000003,"15min":7.9450000000000003},"last":0.56999999999999995}]},{"osd":3,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.627},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.85599999999999998,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.58499999999999996}]},{"osd":4,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.432},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.51600000000000001,"15min":0.50700000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85599999999999998,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60999999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.51900000000000002,"15min":0.51500000000000001},"min":{"1min":0.35499999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77400000000000002,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.497},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.56699999999999995,"15min":0.55900000000000005},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.68200000000000005}]},{"osd":6,"last update":"Fri Apr 26 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.57699999999999996,"15min":0.57099999999999995},"min":{"1min":0.38600000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.58999999999999997,"15min":0.58799999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.55100000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462952,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":901547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.77900000000000003,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.53200000000000003,"15min":0.52900000000000003},"min":{"1min":0.35399999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.61099999999999999}]},{"osd":1,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.442,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76800000000000002,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.67500000000000004}]},{"osd":2,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.376,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54900000000000004,"15min":0.54600000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.64700000000000002}]},{"osd":3,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.53100000000000003,"15min":0.53200000000000003},"min":{"1min":0.35199999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.621},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.57399999999999995,"15min":0.57299999999999995},"min":{"1min":0.432,"5min":0.372,"15min":0.372},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.63300000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.47399999999999998,"15min":0.46400000000000002},"min":{"1min":0.374,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.5,"15min":0.48899999999999999},"min":{"1min":0.35699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.59399999999999997}]},{"osd":5,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.48699999999999999,"15min":0.47799999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.78900000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.50900000000000001,"15min":0.497},"min":{"1min":0.34000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.51200000000000001}]},{"osd":7,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.56299999999999994,"15min":0.55800000000000005},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.79600000000000004,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79200000000000004,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.68799999999999994}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":309292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.373,"15min":0.33300000000000002},"min":{"1min":0.22900000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.58199999999999996,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.35999999999999999},"min":{"1min":0.25,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.31900000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.42099999999999999,"15min":0.41599999999999998},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.48999999999999999,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.71399999999999997,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.40400000000000003}]},{"osd":3,"last update":"Fri Apr 26 04:40:40 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.47099999999999997,"15min":0.48999999999999999},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61199999999999999,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.47199999999999998,"15min":0.498},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.61399999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.45000000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.61899999999999999,"15min":0.627},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.76500000000000001,"5min":1.151,"15min":1.151},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.59399999999999997,"15min":0.60999999999999999},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.69999999999999996,"5min":1,"15min":1},"last":0.64400000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:40:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.60899999999999999,"15min":0.61099999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.93500000000000005,"5min":1.133,"15min":1.133},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.623,"15min":0.63300000000000001},"min":{"1min":0.43099999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.78000000000000003,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.748}]},{"osd":6,"last update":"Fri Apr 26 04:39:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.64500000000000002,"15min":0.65100000000000002},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.65300000000000002,"15min":0.66500000000000004},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.58499999999999996}]},{"osd":7,"last update":"Fri Apr 26 04:40:09 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.63800000000000001,"15min":0.65000000000000002},"min":{"1min":0.46300000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.76200000000000001,"5min":1.069,"15min":1.069},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.628,"15min":0.63800000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68200000000000005,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.70499999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298640,"kb_used_data":1284,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446512,"statfs":{"total":95995035648,"available":95689228288,"internally_reserved":0,"allocated":1314816,"data_stored":900517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 26 04:39:55 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36199999999999999,"15min":0.34799999999999998},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.56299999999999994,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.40500000000000003,"15min":0.38200000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.48799999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:40:13 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.441,"15min":0.42499999999999999},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.42099999999999999,"15min":0.42899999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58599999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.37}]},{"osd":3,"last update":"Fri Apr 26 04:40:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.46300000000000002,"15min":0.47299999999999998},"min":{"1min":0.33700000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70699999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.432}]},{"osd":4,"last update":"Fri Apr 26 04:40:00 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84099999999999997,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.87},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.58999999999999997,"15min":0.59399999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.83999999999999997,"5min":0.872,"15min":0.872},"last":0.86299999999999999}]},{"osd":5,"last update":"Fri Apr 26 04:40:22 2024","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":0.81399999999999995,"15min":0.68600000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":15.811999999999999,"5min":15.811999999999999,"15min":15.811999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.62,"15min":0.626},"min":{"1min":0.44800000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.81799999999999995}]},{"osd":6,"last update":"Fri Apr 26 04:40:49 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.60299999999999998,"15min":0.60099999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.5609999999999999,"5min":0.83199999999999996,"15min":0.71599999999999997},"min":{"1min":0.46800000000000003,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":15.779,"5min":15.779,"15min":15.779},"last":0.72299999999999998}]},{"osd":7,"last update":"Fri Apr 26 04:40:13 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.59399999999999997,"15min":0.58799999999999997},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65400000000000003,"15min":0.65300000000000002},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.67700000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":720896,"data_stored":310161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.48799999999999999,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.37},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.41399999999999998,"15min":0.40999999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.51700000000000002,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.27500000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.42699999999999999,"15min":0.433},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.53500000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.441,"15min":0.443},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.496,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.41799999999999998}]},{"osd":3,"last update":"Fri Apr 26 04:40:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.45500000000000002,"15min":0.47599999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.63600000000000001,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.47499999999999998,"15min":0.49399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65300000000000002,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.38400000000000001}]},{"osd":4,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58999999999999997,"15min":0.60199999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83499999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.64400000000000002,"15min":0.65200000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.64700000000000002}]},{"osd":5,"last update":"Fri Apr 26 04:40:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.61399999999999999,"15min":0.63200000000000001},"min":{"1min":0.41499999999999998,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.83999999999999997,"5min":0.999,"15min":0.999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57599999999999996,"15min":0.59599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.82099999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.67300000000000004}]},{"osd":6,"last update":"Fri Apr 26 04:39:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.61799999999999999,"15min":0.623},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.71999999999999997,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.63700000000000001,"15min":0.64400000000000002},"min":{"1min":0.45000000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60899999999999999}]},{"osd":7,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.58399999999999996,"15min":0.58699999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.75,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.625}]}]},{"osd":3,"up_from":26,"seq":111669149751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":309647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:43 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.40300000000000002,"15min":0.40699999999999997},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.63500000000000001,"5min":1.1799999999999999,"15min":1.1799999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.39900000000000002,"15min":0.39800000000000002},"min":{"1min":0.29099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.60799999999999998,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.34000000000000002}]},{"osd":1,"last update":"Fri Apr 26 04:40:43 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.41399999999999998,"15min":0.40899999999999997},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.79100000000000004,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.308},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.48399999999999999,"15min":0.50600000000000001},"min":{"1min":0.30099999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.77900000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.42699999999999999}]},{"osd":2,"last update":"Fri Apr 26 04:40:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.45000000000000001,"15min":0.44600000000000001},"min":{"1min":0.27800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.68400000000000005,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.442,"15min":0.46400000000000002},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.40799999999999997}]},{"osd":4,"last update":"Fri Apr 26 04:39:58 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58999999999999997,"15min":0.58399999999999996},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.89500000000000002,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.627,"15min":0.65000000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77000000000000002,"5min":1.034,"15min":1.034},"last":0.75}]},{"osd":5,"last update":"Fri Apr 26 04:40:29 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.63900000000000001,"15min":0.65500000000000003},"min":{"1min":0.46300000000000002,"5min":0.373,"15min":0.373},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.628,"15min":0.63900000000000001},"min":{"1min":0.41299999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.80400000000000005,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.72799999999999998}]},{"osd":6,"last update":"Fri Apr 26 04:39:52 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.65800000000000003,"15min":0.66600000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.94899999999999995,"5min":0.998,"15min":0.998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.65300000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:40:10 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.61799999999999999,"15min":0.61599999999999999},"min":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.64800000000000002,"15min":0.65100000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.96299999999999997,"5min":1.014,"15min":1.014},"last":0.63600000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":310990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.014,"15min":1.23},"min":{"1min":0.32000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.76600000000000001,"5min":15.547000000000001,"15min":15.547000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.58899999999999997,"15min":0.61599999999999999},"min":{"1min":0.34000000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.79700000000000004,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.67800000000000005}]},{"osd":1,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.55200000000000005,"15min":0.57199999999999995},"min":{"1min":0.32000000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.72799999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58699999999999997,"15min":0.59699999999999998},"min":{"1min":0.38900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.85299999999999998,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.70999999999999996}]},{"osd":2,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.59599999999999997},"min":{"1min":0.42499999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.755,"5min":0.873,"15min":0.873},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.59599999999999997,"15min":0.61199999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.78000000000000003,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.71899999999999997}]},{"osd":3,"last update":"Fri Apr 26 04:40:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.58499999999999996,"15min":0.59199999999999997},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79200000000000004,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.754},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58899999999999997,"15min":0.60499999999999998},"min":{"1min":0.44400000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78200000000000003,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.69499999999999995}]},{"osd":5,"last update":"Fri Apr 26 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.45800000000000002,"15min":0.442},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.51900000000000002,"15min":0.501},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.51800000000000002}]},{"osd":6,"last update":"Fri Apr 26 04:39:49 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.51200000000000001,"15min":0.501},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64800000000000002}]},{"osd":7,"last update":"Fri Apr 26 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.55700000000000005,"15min":0.54900000000000004},"min":{"1min":0.35699999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.498},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.53200000000000003,"15min":0.51800000000000002},"min":{"1min":0.39000000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.626}]}]},{"osd":5,"up_from":39,"seq":167503724588,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":696,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":712704,"data_stored":309609,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.71599999999999997,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.56899999999999995,"15min":0.57899999999999996},"min":{"1min":0.37,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.67300000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.495}]},{"osd":1,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56599999999999995,"15min":0.57599999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70199999999999996,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.58199999999999996,"15min":0.58899999999999997},"min":{"1min":0.36899999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.72999999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.63400000000000001}]},{"osd":2,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.67800000000000005,"15min":0.61799999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.268000000000001,"5min":11.268000000000001,"15min":11.268000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72599999999999998,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.60999999999999999}]},{"osd":3,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.67200000000000004,"15min":0.62},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":11.24,"5min":11.24,"15min":11.24},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.47099999999999997,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.81200000000000006,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.64500000000000002}]},{"osd":4,"last update":"Fri Apr 26 04:40:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.47799999999999998,"15min":0.46000000000000002},"min":{"1min":0.36799999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.376},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.47999999999999998,"15min":0.45200000000000001},"min":{"1min":0.36899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77500000000000002,"5min":1.244,"15min":1.244},"last":0.53800000000000003}]},{"osd":6,"last update":"Fri Apr 26 04:40:46 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.52500000000000002,"15min":0.499},"min":{"1min":0.41499999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.79600000000000004,"5min":1.109,"15min":1.109},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.52800000000000002,"15min":0.51800000000000002},"min":{"1min":0.38400000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Fri Apr 26 04:40:14 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.55900000000000005,"15min":0.55200000000000005},"min":{"1min":0.40799999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.78600000000000003,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.42599999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92600000000000005,"5min":1.173,"15min":1.173},"last":0.622}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T04:40:53.547 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T04:40:53.547 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T04:40:53.547 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T04:40:53.547 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T04:40:53.816 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:54.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:53] "GET /metrics HTTP/1.1" 200 239041 "" "Prometheus/2.43.0" 2024-04-26T04:40:54.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:54 smithi038 ceph-mon[26454]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:54.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:54 smithi038 ceph-mon[26454]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:54.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:54 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:54.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:54 smithi038 ceph-mon[36743]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:54.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:54 smithi038 ceph-mon[36743]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:54.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:54 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:54 smithi055 ceph-mon[30328]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T04:40:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:54 smithi055 ceph-mon[30328]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:54 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:40:55.583 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T04:40:55.583 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-26T04:40:56.161 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:56 smithi038 ceph-mon[26454]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:56.161 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:56 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1315161574' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:40:56.161 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:56 smithi038 ceph-mon[36743]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:56.161 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:56 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1315161574' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:40:56.189 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-26T04:40:56.190 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:40:56.202 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-26T04:40:56.202 INFO:teuthology.run_tasks:Running task parallel... 2024-04-26T04:40:56.248 INFO:teuthology.task.parallel:starting parallel... 2024-04-26T04:40:56.248 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:40:56.249 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-04-26T04:40:56.249 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-26T04:40:56.251 INFO:tasks.radosbench:Beginning radosbench... 2024-04-26T04:40:56.251 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-04-26T04:40:56.251 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-04-26T04:40:56.251 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-26T04:40:56.252 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-26T04:40:56.252 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi038.front.sepia.ceph.com 2024-04-26T04:40:56.252 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-26T04:40:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:56 smithi055 ceph-mon[30328]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:56 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1315161574' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T04:40:56.545 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:56.549 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:40:57.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:40:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:40:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:40:58.348 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:40:58 smithi038 ceph-mon[26454]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:58.348 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:40:58 smithi038 ceph-mon[36743]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:40:58.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:40:58 smithi055 ceph-mon[30328]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:00.109 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_0' created 2024-04-26T04:41:00.339 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[26454]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:00.339 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/254831209' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-26T04:41:00.339 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' 2024-04-26T04:41:00.339 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[36743]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/254831209' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:40:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:40:59] ENGINE Bus STOPPING 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:41:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:41:00] ENGINE Bus STOPPED 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:41:00] ENGINE Bus STARTING 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:41:00] ENGINE Serving on http://:::9283 2024-04-26T04:41:00.340 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:41:00] ENGINE Bus STARTED 2024-04-26T04:41:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:00 smithi055 ceph-mon[30328]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:00 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/254831209' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-26T04:41:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:00 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' 2024-04-26T04:41:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:00 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:00.799 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-04-26T04:41:01.211 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-26T04:41:01.310 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:01.433 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/254831209' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-26T04:41:01.433 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[26454]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:41:01.433 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:01.433 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:01.433 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:01.434 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/254831209' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-26T04:41:01.434 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[36743]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:41:01.434 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:01.434 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:01.434 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:01 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:01 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/254831209' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-26T04:41:01.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:01 smithi055 ceph-mon[30328]: osdmap e71: 8 total, 8 up, 8 in 2024-04-26T04:41:01.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:01 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:01.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:01 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:01.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:01 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:02 smithi038 ceph-mon[26454]: pgmap v85: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:02 smithi038 ceph-mon[26454]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:41:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:02 smithi038 ceph-mon[26454]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:41:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:02 smithi038 ceph-mon[36743]: pgmap v85: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:02 smithi038 ceph-mon[36743]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:41:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:02 smithi038 ceph-mon[36743]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:41:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:02 smithi055 ceph-mon[30328]: pgmap v85: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:02 smithi055 ceph-mon[30328]: osdmap e72: 8 total, 8 up, 8 in 2024-04-26T04:41:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:02 smithi055 ceph-mon[30328]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:41:02.846 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:03.159 INFO:teuthology.orchestra.run.smithi038.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-04-26T04:41:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-mon[36743]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:41:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1522840656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:03.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-mon[26454]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:41:03.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1522840656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:03.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:03 smithi055 ceph-mon[30328]: osdmap e73: 8 total, 8 up, 8 in 2024-04-26T04:41:03.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:03 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1522840656' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:03.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:03 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:41:04.101 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:03] "GET /metrics HTTP/1.1" 200 239104 "" "Prometheus/2.43.0" 2024-04-26T04:41:04.359 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:04 smithi038 ceph-mon[26454]: pgmap v88: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:04.360 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:04 smithi038 ceph-mon[26454]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:41:04.360 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:04 smithi038 ceph-mon[26454]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:41:04.360 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:04 smithi038 ceph-mon[36743]: pgmap v88: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:04.360 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:04 smithi038 ceph-mon[36743]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:41:04.360 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:04 smithi038 ceph-mon[36743]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:41:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:04 smithi055 ceph-mon[30328]: pgmap v88: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:04 smithi055 ceph-mon[30328]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:41:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:04 smithi055 ceph-mon[30328]: osdmap e74: 8 total, 8 up, 8 in 2024-04-26T04:41:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:05 smithi038 ceph-mon[26454]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:41:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:05 smithi038 ceph-mon[26454]: Cluster is now healthy 2024-04-26T04:41:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:05 smithi038 ceph-mon[36743]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:41:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:05 smithi038 ceph-mon[36743]: Cluster is now healthy 2024-04-26T04:41:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:05 smithi055 ceph-mon[30328]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:41:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:05 smithi055 ceph-mon[30328]: Cluster is now healthy 2024-04-26T04:41:05.617 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-26T04:41:05.888 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:05.974 DEBUG:teuthology.orchestra.run.smithi038:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-04-26T04:41:05.977 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-26T04:41:05.977 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-26T04:41:06.023 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:hints = 1 2024-04-26T04:41:06.023 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-26T04:41:06.023 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object prefix: benchmark_data_smithi038_77313 2024-04-26T04:41:06.023 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:41:06.023 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 0 0 0 0 0 0 - 0 2024-04-26T04:41:06.306 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:06 smithi038 ceph-mon[36743]: pgmap v90: 89 pgs: 13 unknown, 76 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:06.306 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:06 smithi038 ceph-mon[26454]: pgmap v90: 89 pgs: 13 unknown, 76 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:06.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:06 smithi055 ceph-mon[30328]: pgmap v90: 89 pgs: 13 unknown, 76 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:06.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:41:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:07.023 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 1 16 4158 4142 258.859 258.875 0.00327868 0.00385065 2024-04-26T04:41:08.024 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 2 16 7922 7906 247.043 235.25 0.00392234 0.00403872 2024-04-26T04:41:08.118 INFO:teuthology.orchestra.run.smithi038.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:08 smithi038 ceph-mon[26454]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:08 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:08 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:08 smithi038 ceph-mon[36743]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:08 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:08 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:08 smithi055 ceph-mon[30328]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:08 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:08 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:41:09.024 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 3 16 10847 10831 225.627 182.812 0.00473874 0.00442597 2024-04-26T04:41:09.467 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[26454]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:09.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[26454]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:09.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:09.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:09.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:09.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:09.469 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[36743]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:09.470 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[36743]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:09.470 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:09.470 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:09.470 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:09.470 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:09.524 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:41:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:09 smithi055 ceph-mon[30328]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:09 smithi055 ceph-mon[30328]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:41:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:41:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:09 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:41:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:10.024 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 4 16 13973 13957 218.059 195.375 0.00372733 0.00457948 2024-04-26T04:41:10.046 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:10 smithi038 ceph-mon[36743]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:10 smithi038 ceph-mon[36743]: pgmap v92: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:10 smithi038 ceph-mon[26454]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:10 smithi038 ceph-mon[26454]: pgmap v92: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:10 smithi055 ceph-mon[30328]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b22e2ebdeb24376882b7bda2a7329c8cccc2276a 2024-04-26T04:41:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:10 smithi055 ceph-mon[30328]: pgmap v92: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:41:11.024 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 5 16 17091 17075 213.419 194.875 0.0057709 0.00467985 2024-04-26T04:41:12.025 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 6 15 19789 19774 205.925 168.688 0.00455371 0.00485053 2024-04-26T04:41:12.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:12 smithi038 ceph-mon[26454]: pgmap v93: 89 pgs: 89 active+clean; 871 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-26T04:41:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:12 smithi038 ceph-mon[36743]: pgmap v93: 89 pgs: 89 active+clean; 871 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-26T04:41:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:12 smithi055 ceph-mon[30328]: pgmap v93: 89 pgs: 89 active+clean; 871 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:alertmanager.a smithi038 *:9093,9094 running (86s) 25s ago 2m 16.5M - 0.25.0 c8568f914cd2 72d5725ea40b 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:grafana.a smithi055 *:3000 running (112s) 54s ago 112s 82.2M - 9.4.7 954c08fa6188 27b61261d3f1 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi038.qsffiu smithi038 running (66s) 25s ago 66s 13.4M - 17.2.7-902-gc61472c8 0de407c1b3af 841ef3f1e866 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi055.cuaegn smithi055 running (64s) 54s ago 64s 11.0M - 17.2.7-902-gc61472c8 0de407c1b3af 030d9ab50e92 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.x smithi055 *:8443 running (6m) 54s ago 6m 431M - 17.2.7-902-gc61472c8 0de407c1b3af 260c8de4d6a2 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.y smithi038 *:9283 running (8m) 25s ago 8m 477M - 17.2.7-902-gc61472c8 0de407c1b3af 930235385ef7 2024-04-26T04:41:12.742 INFO:teuthology.orchestra.run.smithi038.stdout:mon.a smithi038 running (8m) 25s ago 8m 41.4M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 0e262e788447 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:mon.b smithi055 running (6m) 54s ago 6m 40.9M 2048M 17.2.7-902-gc61472c8 0de407c1b3af e2ab87d945f4 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:mon.c smithi038 running (6m) 25s ago 6m 38.9M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 96c89d482073 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.a smithi038 *:9100 running (2m) 25s ago 2m 16.0M - 1.5.0 0da6a335fe13 b9188809f937 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.b smithi055 *:9100 running (2m) 54s ago 2m 14.0M - 1.5.0 0da6a335fe13 648dbb499184 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.0 smithi038 running (6m) 25s ago 6m 44.6M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 7a0711df145c 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.1 smithi038 running (5m) 25s ago 5m 42.7M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 54e6c9519bdd 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.2 smithi038 running (5m) 25s ago 5m 42.6M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 05094c28a4cd 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.3 smithi038 running (4m) 25s ago 4m 39.5M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 61ce04224439 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.4 smithi055 running (4m) 54s ago 4m 39.0M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 40ece9b7ce4f 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.5 smithi055 running (3m) 54s ago 3m 42.3M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 50fe68fbb905 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.6 smithi055 running (3m) 54s ago 3m 40.4M 4017M 17.2.7-902-gc61472c8 0de407c1b3af bc76cfe8fbcd 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:osd.7 smithi055 running (3m) 54s ago 3m 40.4M 4017M 17.2.7-902-gc61472c8 0de407c1b3af 6ca051d18a67 2024-04-26T04:41:12.743 INFO:teuthology.orchestra.run.smithi038.stdout:prometheus.a smithi055 *:9095 running (78s) 54s ago 2m 33.4M - 2.43.0 a07b618ecd1d f320c6b37648 2024-04-26T04:41:13.025 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 7 16 22285 22269 198.783 155.938 0.00238245 0.00502337 2024-04-26T04:41:13.565 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-26T04:41:14.026 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 8 16 24750 24734 193.18 154.062 0.00498635 0.00516976 2024-04-26T04:41:14.067 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:14.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:13] "GET /metrics HTTP/1.1" 200 241421 "" "Prometheus/2.43.0" 2024-04-26T04:41:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:14 smithi038 ceph-mon[26454]: from='client.24731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:14 smithi038 ceph-mon[26454]: pgmap v94: 89 pgs: 89 active+clean; 871 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:41:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:14 smithi038 ceph-mon[36743]: from='client.24731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:14 smithi038 ceph-mon[36743]: pgmap v94: 89 pgs: 89 active+clean; 871 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:41:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:14 smithi055 ceph-mon[30328]: from='client.24731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:41:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:14 smithi055 ceph-mon[30328]: pgmap v94: 89 pgs: 89 active+clean; 871 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:41:15.026 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 9 16 26957 26941 187.04 137.938 0.00552759 0.00534083 2024-04-26T04:41:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:15 smithi038 ceph-mon[26454]: pgmap v95: 89 pgs: 89 active+clean; 923 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-26T04:41:15.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:15 smithi038 ceph-mon[36743]: pgmap v95: 89 pgs: 89 active+clean; 923 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-26T04:41:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:15 smithi055 ceph-mon[30328]: pgmap v95: 89 pgs: 89 active+clean; 923 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-26T04:41:16.026 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 10 16 29181 29165 182.235 139 0.00538995 0.00548217 2024-04-26T04:41:17.026 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 11 16 31126 31110 176.713 121.562 0.00829069 0.00565339 2024-04-26T04:41:17.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:41:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:17.869 INFO:teuthology.orchestra.run.smithi038.stdout:wait for minority of mons to upgrade 2024-04-26T04:41:18.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 12 15 33068 33053 172.107 121.438 0.0163459 0.00580479 2024-04-26T04:41:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:18 smithi055 ceph-mon[30328]: pgmap v96: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:41:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:18 smithi038 ceph-mon[26454]: pgmap v96: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:41:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:18 smithi038 ceph-mon[36743]: pgmap v96: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:41:18.606 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-26T04:41:19.026 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 13 16 34835 34819 167.358 110.375 0.00966492 0.0059688 2024-04-26T04:41:19.125 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:41:20.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 14 16 36556 36540 163.087 107.562 0.00711727 0.00612705 2024-04-26T04:41:20.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:20 smithi038 ceph-mon[26454]: pgmap v97: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:41:20.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:20 smithi038 ceph-mon[36743]: pgmap v97: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:41:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:20 smithi055 ceph-mon[30328]: pgmap v97: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:41:21.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 15 16 38686 38670 161.089 133.125 0.00708135 0.00620278 2024-04-26T04:41:22.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 16 16 40773 40757 159.173 130.438 0.01031 0.00627812 2024-04-26T04:41:22.229 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:22 smithi038 ceph-mon[26454]: pgmap v98: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 190 MiB/s wr, 3.04k op/s 2024-04-26T04:41:22.229 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:22 smithi038 ceph-mon[36743]: pgmap v98: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 190 MiB/s wr, 3.04k op/s 2024-04-26T04:41:22.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:22 smithi055 ceph-mon[30328]: pgmap v98: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 190 MiB/s wr, 3.04k op/s 2024-04-26T04:41:23.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 17 15 42789 42774 157.225 126.062 0.00486798 0.00635515 2024-04-26T04:41:24.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 18 16 44722 44706 155.198 120.75 0.00726674 0.00643891 2024-04-26T04:41:24.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:23 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:23] "GET /metrics HTTP/1.1" 200 241421 "" "Prometheus/2.43.0" 2024-04-26T04:41:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:24 smithi038 ceph-mon[26454]: pgmap v99: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2024-04-26T04:41:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:24 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1350926611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:24 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:24.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:24 smithi038 ceph-mon[36743]: pgmap v99: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2024-04-26T04:41:24.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:24 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1350926611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:24.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:24 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:24 smithi055 ceph-mon[30328]: pgmap v99: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2024-04-26T04:41:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:24 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1350926611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:24 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:25.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 19 16 46732 46716 153.641 125.625 0.00627516 0.00650407 2024-04-26T04:41:26.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:41:26.027453+0000 min lat: 0.00134765 max lat: 0.133711 avg lat: 0.00657964 2024-04-26T04:41:26.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:41:26.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 20 16 48620 48604 151.859 118 0.00797194 0.00657964 2024-04-26T04:41:26.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:26 smithi055 ceph-mon[30328]: pgmap v100: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-04-26T04:41:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:26 smithi038 ceph-mon[26454]: pgmap v100: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-04-26T04:41:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:26 smithi038 ceph-mon[36743]: pgmap v100: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-04-26T04:41:27.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 21 16 50513 50497 150.261 118.312 0.00619208 0.00665031 2024-04-26T04:41:27.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:41:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:27.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2969543990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:27.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:27 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2969543990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:27 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2969543990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:28.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 22 15 52437 52422 148.899 120.312 0.0125852 0.00671147 2024-04-26T04:41:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:28 smithi038 ceph-mon[26454]: pgmap v101: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-04-26T04:41:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:28 smithi038 ceph-mon[36743]: pgmap v101: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-04-26T04:41:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:28 smithi055 ceph-mon[30328]: pgmap v101: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 166 MiB/s wr, 2.66k op/s 2024-04-26T04:41:29.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 23 16 54220 54204 147.267 111.375 0.00939732 0.00678522 2024-04-26T04:41:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:29 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2364228647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:29 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2364228647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:29 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2364228647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:30.027 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 24 16 56026 56010 145.834 112.875 0.00847028 0.00685263 2024-04-26T04:41:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:30 smithi038 ceph-mon[26454]: pgmap v102: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-04-26T04:41:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:30 smithi038 ceph-mon[36743]: pgmap v102: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-04-26T04:41:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:30 smithi055 ceph-mon[30328]: pgmap v102: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2024-04-26T04:41:31.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 25 16 57832 57816 144.515 112.875 0.0105134 0.00691485 2024-04-26T04:41:32.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 26 16 59515 59499 143.002 105.188 0.0121321 0.00698847 2024-04-26T04:41:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:32 smithi038 ceph-mon[26454]: pgmap v103: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-04-26T04:41:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:32 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3619346442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:32 smithi038 ceph-mon[36743]: pgmap v103: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-04-26T04:41:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:32 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3619346442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:32 smithi055 ceph-mon[30328]: pgmap v103: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-04-26T04:41:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:32 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3619346442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:33.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 27 16 61236 61220 141.69 107.562 0.0244637 0.00705222 2024-04-26T04:41:34.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 28 15 62882 62867 140.305 102.938 0.00842594 0.0071227 2024-04-26T04:41:34.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:34 smithi038 ceph-mon[26454]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-04-26T04:41:34.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:33] "GET /metrics HTTP/1.1" 200 242413 "" "Prometheus/2.43.0" 2024-04-26T04:41:34.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:34 smithi038 ceph-mon[36743]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-04-26T04:41:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:34 smithi055 ceph-mon[30328]: pgmap v104: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-04-26T04:41:35.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 29 16 64465 64449 138.877 98.875 0.00957684 0.00719563 2024-04-26T04:41:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:35 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1155221885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:35.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:35 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1155221885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:35 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1155221885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:36.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 30 16 66105 66089 137.664 102.5 0.0072559 0.00725915 2024-04-26T04:41:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:36 smithi038 ceph-mon[26454]: pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-26T04:41:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:36 smithi038 ceph-mon[36743]: pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-26T04:41:36.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:36 smithi055 ceph-mon[30328]: pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-04-26T04:41:36.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:41:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:37.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 31 16 67650 67634 136.338 96.5625 0.00740729 0.00732994 2024-04-26T04:41:38.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 32 16 69168 69152 135.042 94.875 0.00863533 0.00740083 2024-04-26T04:41:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:38 smithi038 ceph-mon[26454]: pgmap v106: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-04-26T04:41:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4129146865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:38 smithi038 ceph-mon[36743]: pgmap v106: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-04-26T04:41:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4129146865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:38 smithi055 ceph-mon[30328]: pgmap v106: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-04-26T04:41:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4129146865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:39.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 33 16 70729 70713 133.906 97.5625 0.0109178 0.00746269 2024-04-26T04:41:39.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:40.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 34 16 72168 72152 132.612 89.9375 0.0169223 0.00753581 2024-04-26T04:41:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:40 smithi038 ceph-mon[26454]: pgmap v107: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-04-26T04:41:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:40 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/286419390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:40 smithi038 ceph-mon[36743]: pgmap v107: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-04-26T04:41:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:40 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/286419390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:40 smithi055 ceph-mon[30328]: pgmap v107: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-04-26T04:41:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:40 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/286419390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:41.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 35 16 73667 73651 131.5 93.6875 0.0109953 0.00759984 2024-04-26T04:41:42.028 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 36 16 75028 75012 130.21 85.0625 0.0043743 0.00767445 2024-04-26T04:41:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:42 smithi038 ceph-mon[26454]: pgmap v108: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-04-26T04:41:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:42 smithi038 ceph-mon[36743]: pgmap v108: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-04-26T04:41:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:42 smithi055 ceph-mon[30328]: pgmap v108: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-04-26T04:41:43.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 37 16 76310 76294 128.856 80.125 0.0163135 0.00775593 2024-04-26T04:41:43.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:43 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/115025440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:43 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/115025440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:43.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:43 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/115025440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:44.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 38 16 77764 77748 127.857 90.875 0.00768684 0.00781673 2024-04-26T04:41:44.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:43] "GET /metrics HTTP/1.1" 200 242544 "" "Prometheus/2.43.0" 2024-04-26T04:41:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:44 smithi038 ceph-mon[26454]: pgmap v109: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-04-26T04:41:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:44 smithi038 ceph-mon[36743]: pgmap v109: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-04-26T04:41:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:44 smithi055 ceph-mon[30328]: pgmap v109: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-04-26T04:41:45.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 39 15 79186 79171 126.859 88.9375 0.00833342 0.00787793 2024-04-26T04:41:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:45 smithi055 ceph-mon[30328]: pgmap v110: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:41:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:45 smithi038 ceph-mon[26454]: pgmap v110: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:41:45.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:45 smithi038 ceph-mon[36743]: pgmap v110: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:41:46.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:41:46.029279+0000 min lat: 0.00134765 max lat: 0.133711 avg lat: 0.00794521 2024-04-26T04:41:46.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:41:46.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 40 15 80533 80518 125.792 84.1875 0.00747283 0.00794521 2024-04-26T04:41:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:46 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/114460116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:46 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/114460116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:46 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/114460116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:47.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 41 16 81911 81895 124.822 86.0625 0.0156053 0.00800671 2024-04-26T04:41:47.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:41:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:47 smithi055 ceph-mon[30328]: pgmap v111: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-04-26T04:41:47.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:47 smithi038 ceph-mon[26454]: pgmap v111: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-04-26T04:41:47.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:47 smithi038 ceph-mon[36743]: pgmap v111: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-04-26T04:41:48.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 42 16 83231 83215 123.815 82.5 0.00547068 0.00807154 2024-04-26T04:41:48.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:48 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/264018794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:48.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:48 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/264018794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:48 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/264018794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:49.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 43 16 84540 84524 122.838 81.8125 0.0103129 0.00813481 2024-04-26T04:41:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:49 smithi038 ceph-mon[26454]: pgmap v112: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:41:49.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:49 smithi038 ceph-mon[36743]: pgmap v112: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:41:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:49 smithi055 ceph-mon[30328]: pgmap v112: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:41:50.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 44 16 85860 85844 121.921 82.5 0.00768589 0.00819745 2024-04-26T04:41:51.029 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 45 16 87132 87116 120.978 79.5 0.00938019 0.00826149 2024-04-26T04:41:52.031 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 46 16 88353 88337 120.007 76.3125 0.00707189 0.0083282 2024-04-26T04:41:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:52 smithi055 ceph-mon[30328]: pgmap v113: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2024-04-26T04:41:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:52 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3827283815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:52 smithi038 ceph-mon[26454]: pgmap v113: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2024-04-26T04:41:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:52 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3827283815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:52 smithi038 ceph-mon[36743]: pgmap v113: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2024-04-26T04:41:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:52 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3827283815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:53.031 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 47 16 89565 89549 119.064 75.75 0.00590878 0.00839391 2024-04-26T04:41:54.031 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 48 16 90648 90632 117.993 67.6875 0.0117788 0.00847022 2024-04-26T04:41:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:54 smithi055 ceph-mon[30328]: pgmap v114: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-26T04:41:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:54 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1736968832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:54 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:54.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:54 smithi038 ceph-mon[26454]: pgmap v114: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-26T04:41:54.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:54 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1736968832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:54.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:54 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:54.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:41:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:53] "GET /metrics HTTP/1.1" 200 242544 "" "Prometheus/2.43.0" 2024-04-26T04:41:54.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:54 smithi038 ceph-mon[36743]: pgmap v114: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-04-26T04:41:54.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:54 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1736968832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:54.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:54 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:41:55.030 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 49 16 91722 91706 116.955 67.125 0.0103728 0.00854562 2024-04-26T04:41:56.031 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 50 16 92841 92825 116.014 69.9375 0.0183832 0.00861416 2024-04-26T04:41:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:56 smithi055 ceph-mon[30328]: pgmap v115: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:41:56.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:56 smithi038 ceph-mon[26454]: pgmap v115: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:41:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:56 smithi038 ceph-mon[36743]: pgmap v115: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:41:57.031 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 51 16 93906 93890 115.045 66.5625 0.0175288 0.0086878 2024-04-26T04:41:57.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:41:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:41:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:41:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:57 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1013447671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:57.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:57 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1013447671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:57.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:57 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1013447671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:41:58.031 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 52 16 95045 95029 114.2 71.1875 0.00708126 0.00875075 2024-04-26T04:41:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:41:58 smithi038 ceph-mon[26454]: pgmap v116: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-26T04:41:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:41:58 smithi038 ceph-mon[36743]: pgmap v116: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-26T04:41:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:41:58 smithi055 ceph-mon[30328]: pgmap v116: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-04-26T04:41:59.032 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 53 16 96091 96075 113.277 65.375 0.0118906 0.00882312 2024-04-26T04:42:00.032 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 54 16 97105 97089 112.353 63.375 0.0197874 0.00889598 2024-04-26T04:42:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:00 smithi038 ceph-mon[26454]: pgmap v117: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-26T04:42:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:00 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/722442379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:00 smithi038 ceph-mon[36743]: pgmap v117: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-26T04:42:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:00 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/722442379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:00 smithi055 ceph-mon[30328]: pgmap v117: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-04-26T04:42:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:00 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/722442379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:01.032 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 55 16 98210 98194 111.566 69.0625 0.0198099 0.00895845 2024-04-26T04:42:02.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 56 16 99113 99097 110.581 56.4375 0.0152422 0.00903743 2024-04-26T04:42:02.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:02 smithi038 ceph-mon[36743]: pgmap v118: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:42:02.477 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:02 smithi038 ceph-mon[26454]: pgmap v118: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:42:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:02 smithi055 ceph-mon[30328]: pgmap v118: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:42:03.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 57 16 100171 100155 109.801 66.125 0.00699093 0.00910208 2024-04-26T04:42:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:03 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/279511596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:03 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/279511596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:03 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/279511596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:04.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 58 16 101272 101256 109.094 68.8125 0.00757346 0.00916128 2024-04-26T04:42:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:04 smithi038 ceph-mon[26454]: pgmap v119: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-04-26T04:42:04.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:42:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:03] "GET /metrics HTTP/1.1" 200 242610 "" "Prometheus/2.43.0" 2024-04-26T04:42:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:04 smithi038 ceph-mon[36743]: pgmap v119: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-04-26T04:42:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:04 smithi055 ceph-mon[30328]: pgmap v119: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-04-26T04:42:05.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 59 16 102387 102371 108.426 69.6875 0.0116007 0.00921803 2024-04-26T04:42:05.418 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:05 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1937065331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:05.419 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:05 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1937065331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:05 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1937065331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:06.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:42:06.033198+0000 min lat: 0.00134765 max lat: 0.133711 avg lat: 0.00926836 2024-04-26T04:42:06.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:42:06.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 60 16 103551 103535 107.832 72.75 0.00608256 0.00926836 2024-04-26T04:42:06.423 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:06 smithi038 ceph-mon[26454]: pgmap v120: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2024-04-26T04:42:06.423 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:06 smithi038 ceph-mon[36743]: pgmap v120: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2024-04-26T04:42:06.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:06 smithi055 ceph-mon[30328]: pgmap v120: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2024-04-26T04:42:06.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:42:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:07.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 61 16 104697 104681 107.238 71.625 0.0193352 0.0093201 2024-04-26T04:42:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:42:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:42:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:42:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:07.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: pgmap v121: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi055 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2489276293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: pgmap v121: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-04-26T04:42:07.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi055 2024-04-26T04:42:07.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:07 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2489276293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:08.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 62 16 105742 105726 106.562 65.3125 0.0153581 0.00937921 2024-04-26T04:42:08.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:42:08.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: Upgrade: Target is version 19.0.0-2447-gb22e2ebd (unknown) 2024-04-26T04:42:08.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6'] 2024-04-26T04:42:08.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:42:08.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:42:08.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-26T04:42:08.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: pgmap v121: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-04-26T04:42:08.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 on smithi055 2024-04-26T04:42:08.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:07 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2489276293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:09.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 63 16 106871 106855 105.99 70.5625 0.0167633 0.00942978 2024-04-26T04:42:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:09.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:10.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 64 16 107968 107952 105.405 68.5625 0.0157844 0.0094825 2024-04-26T04:42:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:10 smithi055 ceph-mon[30328]: pgmap v122: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-04-26T04:42:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:10 smithi038 ceph-mon[26454]: pgmap v122: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-04-26T04:42:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:10 smithi038 ceph-mon[36743]: pgmap v122: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-04-26T04:42:11.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 65 16 109063 109047 104.837 68.4375 0.0214756 0.00953372 2024-04-26T04:42:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:11 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/688491959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:11 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/688491959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:11.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:11 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/688491959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:12.033 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 66 16 110149 110133 104.277 67.875 0.0239012 0.00958522 2024-04-26T04:42:12.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:12 smithi038 ceph-mon[26454]: pgmap v123: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-26T04:42:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:12 smithi038 ceph-mon[36743]: pgmap v123: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-26T04:42:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:12 smithi055 ceph-mon[30328]: pgmap v123: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-26T04:42:13.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 67 16 111193 111177 103.694 65.25 0.0167205 0.0096386 2024-04-26T04:42:13.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:13 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3899634523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:13.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:13 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3899634523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:13.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:13 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3899634523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:14.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 68 16 112271 112255 103.16 67.375 0.0202532 0.00968889 2024-04-26T04:42:14.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:42:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:13] "GET /metrics HTTP/1.1" 200 242631 "" "Prometheus/2.43.0" 2024-04-26T04:42:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:14 smithi038 ceph-mon[26454]: pgmap v124: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-04-26T04:42:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:14 smithi038 ceph-mon[36743]: pgmap v124: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-04-26T04:42:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:14 smithi055 ceph-mon[30328]: pgmap v124: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-04-26T04:42:15.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 69 16 113324 113308 102.619 65.8125 0.00510007 0.00973983 2024-04-26T04:42:16.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 70 16 114348 114332 102.067 64 0.00575481 0.00979273 2024-04-26T04:42:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:16 smithi038 ceph-mon[26454]: pgmap v125: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-04-26T04:42:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:16 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1111302321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:16 smithi038 ceph-mon[36743]: pgmap v125: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-04-26T04:42:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:16 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1111302321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:16 smithi055 ceph-mon[30328]: pgmap v125: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-04-26T04:42:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:16 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1111302321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:17.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 71 16 115380 115364 101.538 64.5 0.00790409 0.00984364 2024-04-26T04:42:17.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:42:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:18.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 72 16 116410 116394 101.021 64.375 0.0110124 0.00989398 2024-04-26T04:42:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:18 smithi038 ceph-mon[26454]: pgmap v126: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-26T04:42:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:18 smithi038 ceph-mon[36743]: pgmap v126: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-26T04:42:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:18 smithi055 ceph-mon[30328]: pgmap v126: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-04-26T04:42:19.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 73 16 117413 117397 100.496 62.6875 0.0172157 0.00994571 2024-04-26T04:42:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:19 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1269144750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:19 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1269144750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:19 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1269144750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:20.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 74 16 118394 118378 99.9668 61.3125 0.0262505 0.00999829 2024-04-26T04:42:20.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:20 smithi038 ceph-mon[26454]: pgmap v127: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-04-26T04:42:20.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:20 smithi038 ceph-mon[36743]: pgmap v127: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-04-26T04:42:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:20 smithi055 ceph-mon[30328]: pgmap v127: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-04-26T04:42:21.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 75 16 119306 119290 99.3938 57 0.0107431 0.0100559 2024-04-26T04:42:22.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 76 16 120296 120280 98.9001 61.875 0.0096655 0.0101067 2024-04-26T04:42:22.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:22 smithi038 ceph-mon[26454]: pgmap v128: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-26T04:42:22.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:22 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1774605861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:22.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:22 smithi038 ceph-mon[36743]: pgmap v128: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-26T04:42:22.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:22 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1774605861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:22 smithi055 ceph-mon[30328]: pgmap v128: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-26T04:42:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:22 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1774605861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:23.034 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 77 16 121291 121275 98.4233 62.1875 0.0127707 0.010155 2024-04-26T04:42:24.044 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 78 16 122257 122241 97.9355 60.375 0.0155321 0.0102057 2024-04-26T04:42:24.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:42:23 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:23] "GET /metrics HTTP/1.1" 200 242631 "" "Prometheus/2.43.0" 2024-04-26T04:42:24.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:24 smithi038 ceph-mon[26454]: pgmap v129: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2024-04-26T04:42:24.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:24 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2720283485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:24.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:24 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:24.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:24 smithi038 ceph-mon[36743]: pgmap v129: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2024-04-26T04:42:24.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:24 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2720283485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:24.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:24 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:24.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:24 smithi055 ceph-mon[30328]: pgmap v129: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2024-04-26T04:42:24.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:24 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2720283485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:24.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:24 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:25.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 79 16 123221 123205 97.4584 60.25 0.00977266 0.0102558 2024-04-26T04:42:26.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:42:26.035026+0000 min lat: 0.00134765 max lat: 0.133711 avg lat: 0.0103036 2024-04-26T04:42:26.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:42:26.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 80 16 124198 124182 97.0034 61.0625 0.00823178 0.0103036 2024-04-26T04:42:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:26 smithi038 ceph-mon[26454]: pgmap v130: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-04-26T04:42:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:26 smithi038 ceph-mon[36743]: pgmap v130: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-04-26T04:42:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:26 smithi055 ceph-mon[30328]: pgmap v130: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-04-26T04:42:27.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 81 16 125136 125120 96.5295 58.625 0.010493 0.0103538 2024-04-26T04:42:27.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:42:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:27.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3964660371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:27.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:27 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3964660371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:27 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3964660371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:28.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 82 16 126129 126113 96.1091 62.0625 0.00999713 0.0103995 2024-04-26T04:42:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:28 smithi038 ceph-mon[26454]: pgmap v131: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-26T04:42:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:28 smithi038 ceph-mon[36743]: pgmap v131: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-26T04:42:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:28 smithi055 ceph-mon[30328]: pgmap v131: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-26T04:42:29.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 83 16 127034 127018 95.6326 56.5625 0.0115523 0.0104516 2024-04-26T04:42:30.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 84 16 127954 127938 95.1786 57.5 0.00721801 0.0105018 2024-04-26T04:42:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:30 smithi038 ceph-mon[36743]: pgmap v132: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-04-26T04:42:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:30 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/167654937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:30.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:30 smithi038 ceph-mon[26454]: pgmap v132: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-04-26T04:42:30.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/167654937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:30 smithi055 ceph-mon[30328]: pgmap v132: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-04-26T04:42:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/167654937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:31.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 85 16 128912 128896 94.7632 59.875 0.0102028 0.0105475 2024-04-26T04:42:32.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 86 16 129831 129815 94.3292 57.4375 0.0123679 0.0105964 2024-04-26T04:42:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:32 smithi038 ceph-mon[36743]: pgmap v133: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-26T04:42:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:32 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1037440983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:32 smithi038 ceph-mon[26454]: pgmap v133: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-26T04:42:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:32 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1037440983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:32 smithi055 ceph-mon[30328]: pgmap v133: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-26T04:42:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:32 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1037440983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:33.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 87 16 130776 130760 93.9238 59.0625 0.0263013 0.0106419 2024-04-26T04:42:34.035 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 88 16 131702 131686 93.5141 57.875 0.0117843 0.0106889 2024-04-26T04:42:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:34 smithi038 ceph-mon[26454]: pgmap v134: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s 2024-04-26T04:42:34.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:42:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:33] "GET /metrics HTTP/1.1" 200 242658 "" "Prometheus/2.43.0" 2024-04-26T04:42:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:34 smithi038 ceph-mon[36743]: pgmap v134: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s 2024-04-26T04:42:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:34 smithi055 ceph-mon[30328]: pgmap v134: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 797 op/s 2024-04-26T04:42:35.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 89 16 132598 132582 93.0926 56 0.0176651 0.0107373 2024-04-26T04:42:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:35 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3826267597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:35.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:35 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3826267597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:35 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3826267597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total time run: 90.0105 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total writes made: 133496 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Write size: 65536 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object size: 65536 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Bandwidth (MB/sec): 92.6947 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Bandwidth: 40.4124 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max bandwidth (MB/sec): 258.875 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min bandwidth (MB/sec): 56 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average IOPS: 1483 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev IOPS: 646.598 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max IOPS: 4142 2024-04-26T04:42:36.036 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min IOPS: 896 2024-04-26T04:42:36.037 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average Latency(s): 0.0107839 2024-04-26T04:42:36.037 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Latency(s): 0.00725068 2024-04-26T04:42:36.037 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max latency(s): 0.133711 2024-04-26T04:42:36.037 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min latency(s): 0.00134765 2024-04-26T04:42:36.042 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Cleaning up (deleting benchmark objects) 2024-04-26T04:42:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:36 smithi038 ceph-mon[26454]: pgmap v135: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2024-04-26T04:42:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:36 smithi038 ceph-mon[36743]: pgmap v135: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2024-04-26T04:42:36.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:36 smithi055 ceph-mon[30328]: pgmap v135: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2024-04-26T04:42:36.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:42:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:38 smithi055 ceph-mon[30328]: pgmap v136: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-04-26T04:42:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3480404425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:38.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:38 smithi038 ceph-mon[36743]: pgmap v136: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-04-26T04:42:38.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3480404425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:38.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:38 smithi038 ceph-mon[26454]: pgmap v136: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-04-26T04:42:38.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3480404425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:39.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:39.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:40 smithi055 ceph-mon[30328]: pgmap v137: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-04-26T04:42:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:40 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3973120999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:40 smithi038 ceph-mon[36743]: pgmap v137: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-04-26T04:42:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:40 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3973120999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:40 smithi038 ceph-mon[26454]: pgmap v137: 89 pgs: 89 active+clean; 8.1 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-04-26T04:42:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:40 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3973120999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:41 smithi055 ceph-mon[30328]: pgmap v138: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 53 MiB/s wr, 1.15k op/s 2024-04-26T04:42:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:41 smithi038 ceph-mon[26454]: pgmap v138: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 53 MiB/s wr, 1.15k op/s 2024-04-26T04:42:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:41 smithi038 ceph-mon[36743]: pgmap v138: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 53 MiB/s wr, 1.15k op/s 2024-04-26T04:42:43.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:42 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/264403492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:42 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/264403492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:43.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:42 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/264403492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:43 smithi055 ceph-mon[30328]: pgmap v139: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 752 op/s 2024-04-26T04:42:44.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:42:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:43] "GET /metrics HTTP/1.1" 200 242671 "" "Prometheus/2.43.0" 2024-04-26T04:42:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:43 smithi038 ceph-mon[26454]: pgmap v139: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 752 op/s 2024-04-26T04:42:44.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:43 smithi038 ceph-mon[36743]: pgmap v139: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 752 op/s 2024-04-26T04:42:46.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:46 smithi038 ceph-mon[26454]: pgmap v140: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 774 op/s 2024-04-26T04:42:46.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:46 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1771945939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:46.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:46 smithi038 ceph-mon[36743]: pgmap v140: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 774 op/s 2024-04-26T04:42:46.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:46 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1771945939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:46 smithi055 ceph-mon[30328]: pgmap v140: 89 pgs: 89 active+clean; 7.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 29 MiB/s wr, 774 op/s 2024-04-26T04:42:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:46 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1771945939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:47.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:42:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:48 smithi038 ceph-mon[26454]: pgmap v141: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.10k op/s 2024-04-26T04:42:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:48 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/441282173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:48 smithi038 ceph-mon[36743]: pgmap v141: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.10k op/s 2024-04-26T04:42:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:48 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/441282173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:48 smithi055 ceph-mon[30328]: pgmap v141: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 27 MiB/s wr, 1.10k op/s 2024-04-26T04:42:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:48 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/441282173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:50 smithi038 ceph-mon[26454]: pgmap v142: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 4.4 MiB/s wr, 735 op/s 2024-04-26T04:42:50.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:50 smithi038 ceph-mon[36743]: pgmap v142: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 4.4 MiB/s wr, 735 op/s 2024-04-26T04:42:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:50 smithi055 ceph-mon[30328]: pgmap v142: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 4.4 MiB/s wr, 735 op/s 2024-04-26T04:42:51.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:51 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3528755007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:51.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:51 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3528755007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:51 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3528755007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:52 smithi038 ceph-mon[26454]: pgmap v143: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.4 MiB/s wr, 1.10k op/s 2024-04-26T04:42:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:52 smithi038 ceph-mon[36743]: pgmap v143: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.4 MiB/s wr, 1.10k op/s 2024-04-26T04:42:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:52 smithi055 ceph-mon[30328]: pgmap v143: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.4 MiB/s wr, 1.10k op/s 2024-04-26T04:42:54.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:42:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:53] "GET /metrics HTTP/1.1" 200 242671 "" "Prometheus/2.43.0" 2024-04-26T04:42:54.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:54 smithi038 ceph-mon[26454]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-26T04:42:54.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:54 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4198235495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:54.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:54 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:54.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:54 smithi038 ceph-mon[36743]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-26T04:42:54.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:54 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4198235495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:54.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:54 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:54 smithi055 ceph-mon[30328]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-26T04:42:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:54 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4198235495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:54 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:42:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:56 smithi038 ceph-mon[26454]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2024-04-26T04:42:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:56 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2156773862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:56 smithi038 ceph-mon[36743]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2024-04-26T04:42:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:56 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2156773862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:56 smithi055 ceph-mon[30328]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 763 op/s 2024-04-26T04:42:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:56 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2156773862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:57.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:42:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:42:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:42:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:58 smithi038 ceph-mon[26454]: pgmap v146: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-26T04:42:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:58 smithi038 ceph-mon[36743]: pgmap v146: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-26T04:42:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:58 smithi055 ceph-mon[30328]: pgmap v146: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-26T04:42:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:42:59 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1800197108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:42:59 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1800197108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:42:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:42:59 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1800197108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:00 smithi038 ceph-mon[26454]: pgmap v147: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-04-26T04:43:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:00 smithi038 ceph-mon[36743]: pgmap v147: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-04-26T04:43:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:00 smithi055 ceph-mon[30328]: pgmap v147: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-04-26T04:43:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:02 smithi038 ceph-mon[26454]: pgmap v148: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-26T04:43:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:02 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/823575690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:02 smithi038 ceph-mon[36743]: pgmap v148: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-26T04:43:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:02 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/823575690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:02 smithi055 ceph-mon[30328]: pgmap v148: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-26T04:43:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:02 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/823575690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:04 smithi038 ceph-mon[26454]: pgmap v149: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:43:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:04 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/473823252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:04 smithi038 ceph-mon[36743]: pgmap v149: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:43:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:04 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/473823252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:04.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:03] "GET /metrics HTTP/1.1" 200 242741 "" "Prometheus/2.43.0" 2024-04-26T04:43:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:04 smithi055 ceph-mon[30328]: pgmap v149: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:43:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:04 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/473823252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:06 smithi038 ceph-mon[26454]: pgmap v150: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-26T04:43:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:06 smithi038 ceph-mon[36743]: pgmap v150: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-26T04:43:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:06 smithi055 ceph-mon[30328]: pgmap v150: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-04-26T04:43:07.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[32175]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:43:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:07 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2319091462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:07.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:07 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2319091462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:07 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2319091462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:08 smithi038 ceph-mon[26454]: pgmap v151: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-04-26T04:43:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:08 smithi038 ceph-mon[36743]: pgmap v151: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-04-26T04:43:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:08 smithi055 ceph-mon[30328]: pgmap v151: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-04-26T04:43:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:09 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:09 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:09 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:10 smithi055 ceph-mon[30328]: pgmap v152: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2024-04-26T04:43:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:10 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1934463699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:10 smithi038 ceph-mon[26454]: pgmap v152: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2024-04-26T04:43:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:10 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1934463699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:10 smithi038 ceph-mon[36743]: pgmap v152: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2024-04-26T04:43:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:10 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1934463699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: pgmap v153: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-04-26T04:43:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2869801269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:12.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:43:12.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:43:12.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:43:12.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:12 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: pgmap v153: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2869801269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:43:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: pgmap v153: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2869801269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:43:12.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:12 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:13.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:13 smithi055 ceph-mon[30328]: Upgrade: Updating mgr.x 2024-04-26T04:43:13.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:13 smithi055 ceph-mon[30328]: Deploying daemon mgr.x on smithi055 2024-04-26T04:43:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:13 smithi038 ceph-mon[26454]: Upgrade: Updating mgr.x 2024-04-26T04:43:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:13 smithi038 ceph-mon[26454]: Deploying daemon mgr.x on smithi055 2024-04-26T04:43:13.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:13 smithi038 ceph-mon[36743]: Upgrade: Updating mgr.x 2024-04-26T04:43:13.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:13 smithi038 ceph-mon[36743]: Deploying daemon mgr.x on smithi055 2024-04-26T04:43:14.263 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:13] "GET /metrics HTTP/1.1" 200 242764 "" "Prometheus/2.43.0" 2024-04-26T04:43:14.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:14 smithi055 ceph-mon[30328]: pgmap v154: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2024-04-26T04:43:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:14 smithi038 ceph-mon[26454]: pgmap v154: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2024-04-26T04:43:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:14 smithi038 ceph-mon[36743]: pgmap v154: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2024-04-26T04:43:14.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:14 smithi055 systemd[1]: Stopping Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:43:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:15 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1116041667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:15 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1116041667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:15 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1116041667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:16.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:15 smithi055 podman[64226]: 2024-04-26 04:43:15.740072794 +0000 UTC m=+1.040416678 container died 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T04:43:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:16 smithi055 ceph-mon[30328]: pgmap v155: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-04-26T04:43:16.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:16 smithi038 ceph-mon[26454]: pgmap v155: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-04-26T04:43:16.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:16 smithi038 ceph-mon[36743]: pgmap v155: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2024-04-26T04:43:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:17 smithi055 ceph-mon[30328]: pgmap v156: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-04-26T04:43:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:17 smithi038 ceph-mon[26454]: pgmap v156: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-04-26T04:43:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:17 smithi038 ceph-mon[36743]: pgmap v156: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-04-26T04:43:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:18 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1958697928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:18.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:18 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1958697928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:18.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:18 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1958697928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:19 smithi055 ceph-mon[30328]: pgmap v157: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-26T04:43:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:19 smithi038 ceph-mon[26454]: pgmap v157: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-26T04:43:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:19 smithi038 ceph-mon[36743]: pgmap v157: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-26T04:43:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:20 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1669985289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:20 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1669985289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:20 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1669985289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:21.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:20 smithi055 podman[64226]: 2024-04-26 04:43:20.610491501 +0000 UTC m=+5.910835389 container cleanup 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-26T04:43:21.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:20 smithi055 bash[64226]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:43:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:21 smithi055 ceph-mon[30328]: pgmap v158: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:43:21.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:21 smithi038 ceph-mon[26454]: pgmap v158: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:43:21.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:21 smithi038 ceph-mon[36743]: pgmap v158: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:43:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:22 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3984829447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:22 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3984829447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:23.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:22 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3984829447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:24.014 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:23 smithi055 ceph-mon[30328]: pgmap v159: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-26T04:43:24.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:23 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:23] "GET /metrics HTTP/1.1" 200 242764 "" "Prometheus/2.43.0" 2024-04-26T04:43:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:23 smithi038 ceph-mon[26454]: pgmap v159: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-26T04:43:24.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:23 smithi038 ceph-mon[36743]: pgmap v159: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-26T04:43:24.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:24 smithi055 podman[64238]: 2024-04-26 04:43:24.013794023 +0000 UTC m=+8.263689220 container remove 260c8de4d6a2455a2e0934e4db3947740f88c0041d9613a7b81d61be89d9f882 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c) 2024-04-26T04:43:24.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:24 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:43:24.762 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:24 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:43:24.762 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:24 smithi055 systemd[1]: Stopped Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:43:24.762 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:24 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service: Consumed 19.171s CPU time. 2024-04-26T04:43:24.762 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:24 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:25.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:24 smithi055 systemd[1]: Starting Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:43:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:24 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:24 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:25.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:25 smithi055 podman[64350]: 2024-04-26 04:43:25.169101454 +0000 UTC m=+0.035972036 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:43:26.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:25 smithi055 ceph-mon[30328]: pgmap v160: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-26T04:43:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:25 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1233104156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.107 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:25 smithi055 podman[64350]: 2024-04-26 04:43:25.995481903 +0000 UTC m=+0.862352460 container create f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T04:43:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:25 smithi038 ceph-mon[36743]: pgmap v160: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-26T04:43:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:25 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1233104156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:25 smithi038 ceph-mon[26454]: pgmap v160: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-04-26T04:43:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:25 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1233104156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:26.559 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:26 smithi055 podman[64350]: 2024-04-26 04:43:26.273732992 +0000 UTC m=+1.140603557 container init f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T04:43:26.559 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:26 smithi055 podman[64350]: 2024-04-26 04:43:26.278572388 +0000 UTC m=+1.145442948 container start f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T04:43:26.559 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:26 smithi055 bash[64350]: f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd 2024-04-26T04:43:26.559 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:26 smithi055 systemd[1]: Started Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:43:26.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:26.557+0000 7fc4a5451200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:43:27.300 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:26.919+0000 7fc4a5451200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:27.301 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:27.125+0000 7fc4a5451200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:43:27.573 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:27.573 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:27.573 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:27.573 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-mon[30328]: pgmap v161: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:43:27.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:27.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:27.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:27.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[36743]: pgmap v161: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:43:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:27.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:27.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:27 smithi038 ceph-mon[26454]: pgmap v161: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:43:27.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:27.730+0000 7fc4a5451200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:43:28.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:27.873+0000 7fc4a5451200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:43:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:28 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1372803662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:28 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1372803662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:28 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1372803662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:29 smithi038 ceph-mon[36743]: pgmap v162: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-26T04:43:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:29 smithi038 ceph-mon[26454]: pgmap v162: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-26T04:43:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:29 smithi055 ceph-mon[30328]: pgmap v162: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-26T04:43:30.323 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:30.041+0000 7fc4a5451200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:43:30.590 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:30.319+0000 7fc4a5451200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:30.590 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:30.586+0000 7fc4a5451200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:43:31.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:30 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:31.095 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:30 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:31.095 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2148040003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:30 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:31.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:30 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:31.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2148040003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:30 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:30 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:30 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2148040003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:31.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:31.110+0000 7fc4a5451200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:43:31.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:31.269+0000 7fc4a5451200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:43:31.906 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:31.607+0000 7fc4a5451200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:43:32.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:31 smithi055 ceph-mon[30328]: pgmap v163: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-26T04:43:32.170 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:31.957+0000 7fc4a5451200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:32.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:31 smithi038 ceph-mon[26454]: pgmap v163: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-26T04:43:32.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:31 smithi038 ceph-mon[36743]: pgmap v163: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-26T04:43:33.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:32.991+0000 7fc4a5451200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:43:33.956 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:33.783+0000 7fc4a5451200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:43:33.956 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:33 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:33.956 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:33 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:33.956 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:33 smithi055 ceph-mon[30328]: pgmap v164: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-26T04:43:33.956 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:33 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3807690850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[26454]: pgmap v164: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3807690850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:33.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[36743]: pgmap v164: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-26T04:43:33.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3807690850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:34.342 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:33.954+0000 7fc4a5451200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:43:34.342 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:34.340+0000 7fc4a5451200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:43:34.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:33] "GET /metrics HTTP/1.1" 200 242794 "" "Prometheus/2.43.0" 2024-04-26T04:43:34.904 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:34 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:34.904 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:34 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:35.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:34 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:35.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:34 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:35.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:34 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:35.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:34 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:35.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:35.259+0000 7fc4a5451200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:43:35.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:35.416+0000 7fc4a5451200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:43:35.891 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:35.844+0000 7fc4a5451200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:43:36.220 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:36.093+0000 7fc4a5451200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:43:36.220 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:35 smithi055 ceph-mon[30328]: pgmap v165: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-26T04:43:36.223 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:35 smithi038 ceph-mon[26454]: pgmap v165: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-26T04:43:36.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:35 smithi038 ceph-mon[36743]: pgmap v165: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2024-04-26T04:43:36.486 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:36.267+0000 7fc4a5451200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:43:36.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:36.483+0000 7fc4a5451200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:36.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:36.648+0000 7fc4a5451200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:43:37.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:36 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3394896602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:37.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:36 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3394896602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:37.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:36 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3394896602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:38.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:37 smithi038 ceph-mon[26454]: pgmap v166: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-04-26T04:43:38.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:37 smithi038 ceph-mon[36743]: pgmap v166: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-04-26T04:43:38.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:37 smithi055 ceph-mon[30328]: pgmap v166: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-04-26T04:43:38.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:38.070+0000 7fc4a5451200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:38.454+0000 7fc4a5451200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:38] ENGINE Bus STARTING 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: CherryPy Checker: 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: The Application mounted at '' has an empty config. 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:38] ENGINE Serving on http://:::9283 2024-04-26T04:43:38.752 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:38] ENGINE Bus STARTED 2024-04-26T04:43:39.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: Standby manager daemon x restarted 2024-04-26T04:43:39.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: Standby manager daemon x started 2024-04-26T04:43:39.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:43:39.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:39.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:43:39.094 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:39.095 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2013802377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: Standby manager daemon x restarted 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: Standby manager daemon x started 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2013802377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: Standby manager daemon x restarted 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: Standby manager daemon x started 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:39.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2013802377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:40.070 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: mgrmap e21: y(active, since 4m), standbys: x 2024-04-26T04:43:40.070 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:40.070 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: pgmap v167: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2024-04-26T04:43:40.070 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: Activating manager daemon x 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: mgrmap e22: x(active, starting, since 0.0206646s) 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:43:40.071 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: Manager daemon x is now available 2024-04-26T04:43:40.072 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:40.073 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:40.073 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:39] ENGINE Bus STOPPING 2024-04-26T04:43:40.073 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:43:40.073 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:39] ENGINE Bus STOPPED 2024-04-26T04:43:40.073 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:39] ENGINE Bus STARTING 2024-04-26T04:43:40.191 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: mgrmap e21: y(active, since 4m), standbys: x 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: pgmap v167: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: Activating manager daemon x 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:43:40.192 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: mgrmap e22: x(active, starting, since 0.0206646s) 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:40.193 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: Manager daemon x is now available 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setuser ceph since I am not root 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ignoring --setgroup ceph since I am not root 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mgr[26688]: -- 172.21.15.38:0/2781480678 <== mon.2 v2:172.21.15.38:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x55e170fb6c60 con 0x55e170f79000 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mgr[26688]: -- 172.21.15.38:0/2781480678 <== mon.2 v2:172.21.15.38:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e170fa29c0 con 0x55e170f79000 2024-04-26T04:43:40.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:40.033+0000 7f690a0711c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:43:40.195 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: mgrmap e21: y(active, since 4m), standbys: x 2024-04-26T04:43:40.195 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:40.195 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: pgmap v167: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2024-04-26T04:43:40.195 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 172.21.15.38:0/3822229017' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: Activating manager daemon x 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: osdmap e75: 8 total, 8 up, 8 in 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24332 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: mgrmap e22: x(active, starting, since 0.0206646s) 2024-04-26T04:43:40.196 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:43:40.197 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:43:40.198 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: Manager daemon x is now available 2024-04-26T04:43:40.198 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:43:40.198 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:39 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:40.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:40] ENGINE Serving on http://:::9283 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:43:40] ENGINE Bus STARTED 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.201+0000 7fc40e853700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.201+0000 7fc40e853700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.201+0000 7fc40e853700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.201+0000 7fc40e853700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.201+0000 7fc40e853700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.242+0000 7fc40303c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.242+0000 7fc40303c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.242+0000 7fc40303c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.242+0000 7fc40303c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:43:40.242+0000 7fc40303c700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:43:40.451 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:40.271+0000 7f690a0711c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:40.452 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:40.450+0000 7f690a0711c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:43:41.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:40.966+0000 7f690a0711c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:43:41.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:41.104+0000 7f690a0711c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:43:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[26454]: mgrmap e23: x(active, since 1.01585s) 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:41.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:40 smithi038 ceph-mon[36743]: mgrmap e23: x(active, since 1.01585s) 2024-04-26T04:43:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:43:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-26T04:43:41.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:40 smithi055 ceph-mon[30328]: mgrmap e23: x(active, since 1.01585s) 2024-04-26T04:43:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[26454]: Deploying cephadm binary to smithi038 2024-04-26T04:43:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[26454]: Deploying cephadm binary to smithi055 2024-04-26T04:43:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[26454]: pgmap v3: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1555580878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:42.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[36743]: Deploying cephadm binary to smithi038 2024-04-26T04:43:42.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[36743]: Deploying cephadm binary to smithi055 2024-04-26T04:43:42.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[36743]: pgmap v3: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:42.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:41 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1555580878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:41 smithi055 ceph-mon[30328]: Deploying cephadm binary to smithi038 2024-04-26T04:43:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:41 smithi055 ceph-mon[30328]: Deploying cephadm binary to smithi055 2024-04-26T04:43:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:41 smithi055 ceph-mon[30328]: pgmap v3: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:41 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1555580878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:42.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:43:42.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T04:43:43.112 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:42.853+0000 7f690a0711c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:43:43.112 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-mon[26454]: pgmap v4: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:43.113 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:43.113 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:43.113 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-mon[36743]: pgmap v4: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:43.113 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:43.113 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:42 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:42 smithi055 ceph-mon[30328]: pgmap v4: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:43.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:42 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:43.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:42 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:43.378 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:43.111+0000 7f690a0711c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:43.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:43.377+0000 7f690a0711c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:43:44.218 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:43.879+0000 7f690a0711c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:43:44.218 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:44.048+0000 7f690a0711c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:43:44.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-mon[26454]: mgrmap e24: x(active, since 3s) 2024-04-26T04:43:44.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-mon[26454]: [26/Apr/2024:04:43:43] ENGINE Bus STARTING 2024-04-26T04:43:44.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-mon[36743]: mgrmap e24: x(active, since 3s) 2024-04-26T04:43:44.219 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:43 smithi038 ceph-mon[36743]: [26/Apr/2024:04:43:43] ENGINE Bus STARTING 2024-04-26T04:43:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:43 smithi055 ceph-mon[30328]: mgrmap e24: x(active, since 3s) 2024-04-26T04:43:44.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:43 smithi055 ceph-mon[30328]: [26/Apr/2024:04:43:43] ENGINE Bus STARTING 2024-04-26T04:43:44.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:44.217+0000 7f690a0711c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:43:44.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:44.644+0000 7f690a0711c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:43:45.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:44 smithi055 ceph-mon[30328]: [26/Apr/2024:04:43:43] ENGINE Serving on https://172.21.15.55:7150 2024-04-26T04:43:45.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:44 smithi055 ceph-mon[30328]: [26/Apr/2024:04:43:43] ENGINE Serving on http://172.21.15.55:8765 2024-04-26T04:43:45.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:44 smithi055 ceph-mon[30328]: [26/Apr/2024:04:43:43] ENGINE Bus STARTED 2024-04-26T04:43:45.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:44 smithi055 ceph-mon[30328]: pgmap v5: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:45.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:44 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4246647725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[26454]: [26/Apr/2024:04:43:43] ENGINE Serving on https://172.21.15.55:7150 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[26454]: [26/Apr/2024:04:43:43] ENGINE Serving on http://172.21.15.55:8765 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[26454]: [26/Apr/2024:04:43:43] ENGINE Bus STARTED 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[26454]: pgmap v5: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4246647725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[36743]: [26/Apr/2024:04:43:43] ENGINE Serving on https://172.21.15.55:7150 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[36743]: [26/Apr/2024:04:43:43] ENGINE Serving on http://172.21.15.55:8765 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[36743]: [26/Apr/2024:04:43:43] ENGINE Bus STARTED 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[36743]: pgmap v5: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail 2024-04-26T04:43:45.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:44 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4246647725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:45.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:45 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:45.380+0000 7f690a0711c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:43:46.714 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:46.233+0000 7f690a0711c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:43:46.714 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:46.400+0000 7f690a0711c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:43:46.714 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:46.713+0000 7f690a0711c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:43:47.011 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:46] "GET /metrics HTTP/1.1" 200 33118 "" "Prometheus/2.43.0" 2024-04-26T04:43:47.011 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:47 smithi055 ceph-mon[30328]: pgmap v6: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-26T04:43:47.011 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:47 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2348861219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:47.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:47.220+0000 7f690a0711c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:43:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-mon[26454]: pgmap v6: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-26T04:43:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2348861219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:47.478 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-mon[36743]: pgmap v6: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 161 op/s 2024-04-26T04:43:47.478 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2348861219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:47.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:47.605+0000 7f690a0711c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:43:48.338 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:47.996+0000 7f690a0711c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:43:48.338 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:48 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:48.191+0000 7f690a0711c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:43:48.698 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:48 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:48.336+0000 7f690a0711c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:43:48.698 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:48 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:48.536+0000 7f690a0711c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:43:48.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:48 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:48.695+0000 7f690a0711c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:43:49.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:49 smithi055 ceph-mon[30328]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 546 op/s 2024-04-26T04:43:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:49 smithi038 ceph-mon[26454]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 546 op/s 2024-04-26T04:43:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:49 smithi038 ceph-mon[36743]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 546 op/s 2024-04-26T04:43:50.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:50 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4235798055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:50.357 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4235798055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:50.357 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:50.136+0000 7f690a0711c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:43:50.357 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4235798055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:50.621 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:50.510+0000 7f690a0711c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:43:50.622 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:43:50] ENGINE Bus STARTING 2024-04-26T04:43:50.622 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: CherryPy Checker: 2024-04-26T04:43:50.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: The Application mounted at '' has an empty config. 2024-04-26T04:43:50.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: 2024-04-26T04:43:50.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:43:50] ENGINE Serving on http://:::9283 2024-04-26T04:43:50.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: [26/Apr/2024:04:43:50] ENGINE Bus STARTED 2024-04-26T04:43:51.337 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[36743]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 424 op/s 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[36743]: Standby manager daemon y started 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[26454]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 424 op/s 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[26454]: Standby manager daemon y started 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:51.338 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:43:51.339 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:51 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:51 smithi055 ceph-mon[30328]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 424 op/s 2024-04-26T04:43:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:51 smithi055 ceph-mon[30328]: Standby manager daemon y started 2024-04-26T04:43:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:51 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:43:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:51 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:43:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:51 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:43:51.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:51 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1267471092' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:43:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:52 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:43:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:52 smithi055 ceph-mon[30328]: mgrmap e25: x(active, since 11s), standbys: y 2024-04-26T04:43:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:52 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:43:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:52 smithi038 ceph-mon[26454]: mgrmap e25: x(active, since 11s), standbys: y 2024-04-26T04:43:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:52 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:43:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:52 smithi038 ceph-mon[36743]: mgrmap e25: x(active, since 11s), standbys: y 2024-04-26T04:43:52.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:43:52 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:43:52.322Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi038:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T04:43:53.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:53 smithi038 ceph-mon[26454]: pgmap v9: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 680 op/s 2024-04-26T04:43:53.356 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:53 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2278055373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:53.356 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:53 smithi038 ceph-mon[36743]: pgmap v9: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 680 op/s 2024-04-26T04:43:53.357 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:53 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2278055373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:53.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:53 smithi055 ceph-mon[30328]: pgmap v9: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 680 op/s 2024-04-26T04:43:53.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:53 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2278055373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:54.266 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:43:53 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:43:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:55 smithi055 ceph-mon[30328]: pgmap v10: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 621 op/s 2024-04-26T04:43:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:55 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:55.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[26454]: pgmap v10: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 621 op/s 2024-04-26T04:43:55.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:55.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:55.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:55.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[36743]: pgmap v10: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 621 op/s 2024-04-26T04:43:55.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:55.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:55.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:55 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:43:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:56 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/68173576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:56 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/68173576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:56.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:56 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/68173576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:57.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:43:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:43:56] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.43.0" 2024-04-26T04:43:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: pgmap v11: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 641 op/s 2024-04-26T04:43:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: pgmap v11: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 641 op/s 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:57 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: pgmap v11: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 641 op/s 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:57.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:57 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:58 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi038 to 2481M 2024-04-26T04:43:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:58 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2617155879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:58 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi038 to 2481M 2024-04-26T04:43:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:58 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2617155879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:58 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi038 to 2481M 2024-04-26T04:43:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:58 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2617155879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:43:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:59 smithi038 ceph-mon[26454]: pgmap v12: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2024-04-26T04:43:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:59 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:43:59 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:59 smithi038 ceph-mon[36743]: pgmap v12: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2024-04-26T04:43:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:59 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:43:59 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:59.476 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:59 smithi055 ceph-mon[30328]: pgmap v12: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2024-04-26T04:43:59.476 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:59 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:43:59.476 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:43:59 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:01.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:01 smithi055 ceph-mon[30328]: pgmap v13: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-04-26T04:44:01.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:01 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2288441330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:01.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:01 smithi038 ceph-mon[26454]: pgmap v13: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-04-26T04:44:01.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:01 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2288441330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:01.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:01 smithi038 ceph-mon[36743]: pgmap v13: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2024-04-26T04:44:01.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:01 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2288441330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:03 smithi038 ceph-mon[26454]: pgmap v14: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-04-26T04:44:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:03 smithi038 ceph-mon[36743]: pgmap v14: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-04-26T04:44:03.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:03 smithi055 ceph-mon[30328]: pgmap v14: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-04-26T04:44:04.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:04 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2402738376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:04 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2402738376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:04 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2402738376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: pgmap v15: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Adjusting osd_memory_target on smithi055 to 2481M 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:05.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:05 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: pgmap v15: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-26T04:44:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Adjusting osd_memory_target on smithi055 to 2481M 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:05.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: pgmap v15: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Adjusting osd_memory_target on smithi055 to 2481M 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:05.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:05 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:44:07.023 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:06 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:06] "GET /metrics HTTP/1.1" 200 36055 "" "Prometheus/2.43.0" 2024-04-26T04:44:07.290 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: pgmap v16: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3866356604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.291 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[26454]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: pgmap v16: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3866356604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.292 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:07 smithi038 ceph-mon[36743]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:44:07.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:07.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:44:07.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: pgmap v16: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3866356604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:07.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:07 smithi055 ceph-mon[30328]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-26T04:44:08.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:08 smithi055 ceph-mon[30328]: Reconfiguring daemon alertmanager.a on smithi038 2024-04-26T04:44:08.416 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:08 smithi038 ceph-mon[26454]: Reconfiguring daemon alertmanager.a on smithi038 2024-04-26T04:44:08.416 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:08 smithi038 ceph-mon[36743]: Reconfiguring daemon alertmanager.a on smithi038 2024-04-26T04:44:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:09 smithi055 ceph-mon[30328]: pgmap v17: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:44:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:09 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2935812069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:09 smithi038 ceph-mon[26454]: pgmap v17: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:44:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:09 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2935812069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:09.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:09 smithi038 ceph-mon[36743]: pgmap v17: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:44:09.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:09 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2935812069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:10 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:10.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:10 smithi038 systemd[1]: Stopping Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:10 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:10 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:11.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:11 smithi038 ceph-mon[36743]: pgmap v18: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-04-26T04:44:11.225 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[70419]: ts=2024-04-26T04:44:10.907Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:44:11.225 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:10 smithi038 podman[82115]: 2024-04-26 04:44:10.950376993 +0000 UTC m=+0.486620668 container died 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:11.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:11 smithi038 ceph-mon[26454]: pgmap v18: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-04-26T04:44:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:11 smithi055 ceph-mon[30328]: pgmap v18: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-04-26T04:44:11.535 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:11 smithi038 podman[82115]: 2024-04-26 04:44:11.534622786 +0000 UTC m=+1.070866452 container cleanup 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:11.871 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:11 smithi038 bash[82115]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T04:44:11.871 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:11 smithi038 podman[82141]: 2024-04-26 04:44:11.769977714 +0000 UTC m=+0.817405718 container remove 72d5725ea40b14934796d1c85355d46c3261d9237d299e87ce4d552159d06043 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:12.224 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:11 smithi038 podman[82141]: 2024-04-26 04:44:11.87123674 +0000 UTC m=+0.918664748 volume remove 0c7f9c8766cc04844b6905a4b52accaf8a1a5f3d5738a33c83bc718a91f2f5e1 2024-04-26T04:44:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:12 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3087272361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:12 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3087272361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:12 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3087272361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:12.585 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T04:44:12.585 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 systemd[1]: Stopped Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:12.585 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.358s CPU time. 2024-04-26T04:44:12.585 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 systemd[1]: Starting Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:12.932 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 podman[82229]: 2024-04-26 04:44:12.482276596 +0000 UTC m=+0.033858299 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-26T04:44:12.933 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 podman[82229]: 2024-04-26 04:44:12.81492023 +0000 UTC m=+0.366501911 volume create da727872143e0fe95cc74eeace2ca10c862adb3b76f683def1bc393c4bbeda8a 2024-04-26T04:44:12.933 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:12 smithi038 podman[82229]: 2024-04-26 04:44:12.932318949 +0000 UTC m=+0.483900632 container create bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:13.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-mon[26454]: pgmap v19: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-26T04:44:13.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-mon[36743]: pgmap v19: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-26T04:44:13.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:13 smithi055 ceph-mon[30328]: pgmap v19: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-26T04:44:13.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 podman[82229]: 2024-04-26 04:44:13.269810917 +0000 UTC m=+0.821392602 container init bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:13.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 podman[82229]: 2024-04-26 04:44:13.276795889 +0000 UTC m=+0.828377572 container start bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:13.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.307Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-26T04:44:13.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.307Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-26T04:44:13.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.312Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.38 port=9094 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.317Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.368Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.368Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.372Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:13.372Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 bash[82229]: bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 2024-04-26T04:44:13.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:13 smithi038 systemd[1]: Started Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:14.087 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[26684]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:44:14.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:14.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:14.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[26454]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:44:14.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[26454]: Reconfiguring daemon node-exporter.a on smithi038 2024-04-26T04:44:14.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[26454]: pgmap v20: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2702472684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[36743]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[36743]: Reconfiguring daemon node-exporter.a on smithi038 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[36743]: pgmap v20: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-26T04:44:14.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:14 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2702472684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:14.726 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:14 smithi038 systemd[1]: Stopping Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:14 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:14 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:14 smithi055 ceph-mon[30328]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-26T04:44:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:14 smithi055 ceph-mon[30328]: Reconfiguring daemon node-exporter.a on smithi038 2024-04-26T04:44:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:14 smithi055 ceph-mon[30328]: pgmap v20: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-26T04:44:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:14 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2702472684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:15.224 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:14 smithi038 podman[82404]: 2024-04-26 04:44:14.897114131 +0000 UTC m=+0.167401436 container died b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:15.669 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:15 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:15.323Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.0002575s 2024-04-26T04:44:15.669 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:15 smithi038 podman[82404]: 2024-04-26 04:44:15.442233525 +0000 UTC m=+0.712520828 container cleanup b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:15.670 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:15 smithi038 bash[82404]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T04:44:15.974 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:15 smithi038 podman[82416]: 2024-04-26 04:44:15.669180191 +0000 UTC m=+0.771762211 container remove b9188809f93741b1720744d3e2996021cc7d141525ab18bf571deeffa460d085 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:15.974 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:15 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:16.428 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:16 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:16.429 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:16 smithi038 systemd[1]: Stopped Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:16.429 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:16 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 2.683s CPU time. 2024-04-26T04:44:16.429 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:16 smithi038 systemd[1]: Starting Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:16.724 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:16 smithi038 podman[82495]: 2024-04-26 04:44:16.333687757 +0000 UTC m=+0.038722423 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:44:17.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:16 smithi055 ceph-mon[30328]: pgmap v21: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-26T04:44:17.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:16] "GET /metrics HTTP/1.1" 200 36053 "" "Prometheus/2.43.0" 2024-04-26T04:44:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:16 smithi038 ceph-mon[26454]: pgmap v21: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-26T04:44:17.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:16 smithi038 ceph-mon[36743]: pgmap v21: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 podman[82495]: 2024-04-26 04:44:17.282838378 +0000 UTC m=+0.987873041 container create f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 podman[82495]: 2024-04-26 04:44:17.516123952 +0000 UTC m=+1.221158628 container init f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 podman[82495]: 2024-04-26 04:44:17.520184454 +0000 UTC m=+1.225219118 container start f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.524Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.524Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.524Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.524Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.525Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.525Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:44:17.611 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:44:17.612 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:44:17.613 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:44:17.614 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:44:17.615 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:44:17.615 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:44:17.615 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a[82533]: ts=2024-04-26T04:44:17.526Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:44:17.870 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 bash[82495]: f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f 2024-04-26T04:44:17.870 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 04:44:17 smithi038 systemd[1]: Started Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:18.158 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:17 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1786052338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:18.158 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:17 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:18.159 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:17 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:18.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1786052338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:18.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:18.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:18.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1786052338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:18.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:18.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:17 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:19.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:18 smithi038 ceph-mon[26454]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:44:19.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:18 smithi038 ceph-mon[26454]: pgmap v22: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-26T04:44:19.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:18 smithi038 ceph-mon[26454]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:44:19.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:18 smithi038 ceph-mon[36743]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:44:19.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:18 smithi038 ceph-mon[36743]: pgmap v22: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-26T04:44:19.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:18 smithi038 ceph-mon[36743]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:44:19.288 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:18 smithi055 ceph-mon[30328]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:44:19.288 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:18 smithi055 ceph-mon[30328]: pgmap v22: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-26T04:44:19.288 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:18 smithi055 ceph-mon[30328]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:44:20.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:19 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1655155450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:20.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:19 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1655155450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:19 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1655155450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:21.027 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:20 smithi055 ceph-mon[30328]: pgmap v23: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-04-26T04:44:21.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:20 smithi038 ceph-mon[26454]: pgmap v23: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-04-26T04:44:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:20 smithi038 ceph-mon[36743]: pgmap v23: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-04-26T04:44:22.356 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 systemd[1]: Stopping Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:22.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.598Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:44:22.890 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:44:22.890 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.599Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.600Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.600Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.600Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[61604]: ts=2024-04-26T04:44:22.600Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:44:22.891 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:22 smithi055 podman[72581]: 2024-04-26 04:44:22.608789467 +0000 UTC m=+0.158716108 container died f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:23.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:22 smithi038 ceph-mon[36743]: pgmap v24: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-26T04:44:23.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:22 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2623814584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:22 smithi038 ceph-mon[26454]: pgmap v24: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-26T04:44:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:22 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2623814584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:23.323 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-mon[30328]: pgmap v24: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-26T04:44:23.323 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:22 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2623814584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:23.323 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 podman[72581]: 2024-04-26 04:44:23.029463128 +0000 UTC m=+0.579389767 container cleanup f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:23.324 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 bash[72581]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:44:23.574 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 podman[72593]: 2024-04-26 04:44:23.322875569 +0000 UTC m=+0.704571003 container remove f320c6b37648317046e39760459907bdac4a68c66669bca6bbaf58aba6f15167 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:23.574 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:44:23.574 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 systemd[1]: Stopped Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:23.574 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 2.198s CPU time. 2024-04-26T04:44:23.574 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 systemd[1]: Starting Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:23.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:44:23 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:44:23.328Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.005334204s 2024-04-26T04:44:24.222 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 podman[72664]: 2024-04-26 04:44:23.8138858 +0000 UTC m=+0.047769329 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:44:24.222 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:23 smithi055 podman[72664]: 2024-04-26 04:44:23.941346126 +0000 UTC m=+0.175229682 container create 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:24.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 podman[72664]: 2024-04-26 04:44:24.221579606 +0000 UTC m=+0.455463139 container init 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 podman[72664]: 2024-04-26 04:44:24.232982959 +0000 UTC m=+0.466866483 container start 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.301Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.302Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.302Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi055 (none))" 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.302Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.302Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.305Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.306Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.307Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.311Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.316Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:44:24.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.316Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.542µs 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.317Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.317Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 bash[72664]: 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 systemd[1]: Started Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.358Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.359Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.359Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=42.042µs wal_replay_duration=42.062177ms wbl_replay_duration=152ns total_replay_duration=42.346841ms 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.363Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.363Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.363Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.404Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=40.90776ms db_storage=1.126µs remote_storage=1.485µs web_handler=663ns query_engine=869ns scrape=655.663µs scrape_sd=130.238µs notify=23.769µs notify_sd=14.658µs rules=39.631311ms tracing=7.57µs 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.404Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:44:24.609 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:24.404Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:44:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:24 smithi038 ceph-mon[36743]: pgmap v25: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-26T04:44:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:24 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:25.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:24 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:24 smithi038 ceph-mon[26454]: pgmap v25: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-26T04:44:25.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:24 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:25.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:24 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:25.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-mon[30328]: pgmap v25: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-26T04:44:25.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:25.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:24 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:25.241 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:25 smithi055 systemd[1]: Stopping Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:26.042 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:25 smithi055 ceph-mon[30328]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:44:26.042 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:25 smithi055 ceph-mon[30328]: Reconfiguring daemon node-exporter.b on smithi055 2024-04-26T04:44:26.042 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:25 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:26.042 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:25 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4207543137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:26.042 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:25 smithi055 podman[72806]: 2024-04-26 04:44:25.736964346 +0000 UTC m=+0.186476747 container died 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[36743]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[36743]: Reconfiguring daemon node-exporter.b on smithi055 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4207543137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[26454]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[26454]: Reconfiguring daemon node-exporter.b on smithi055 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:25 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4207543137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:26.356 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 podman[72806]: 2024-04-26 04:44:26.139685233 +0000 UTC m=+0.589197621 container cleanup 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:26.356 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 bash[72806]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T04:44:26.692 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 podman[72818]: 2024-04-26 04:44:26.441624977 +0000 UTC m=+0.706603679 container remove 648dbb499184244db841d1563d1cea86f922c8fa82b9624383ffaea8bd30ac01 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:26.692 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:26.692 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T04:44:26.693 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 systemd[1]: Stopped Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:26.693 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 2.674s CPU time. 2024-04-26T04:44:26.693 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:26 smithi055 systemd[1]: Starting Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:26 smithi038 ceph-mon[26454]: pgmap v26: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-26T04:44:27.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:26 smithi038 ceph-mon[36743]: pgmap v26: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-26T04:44:27.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:26 smithi055 ceph-mon[30328]: pgmap v26: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-04-26T04:44:27.251 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 podman[72896]: 2024-04-26 04:44:26.935863207 +0000 UTC m=+0.033357478 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-26T04:44:27.251 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 podman[72896]: 2024-04-26 04:44:27.109524257 +0000 UTC m=+0.207018503 container create 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:27.608 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 podman[72896]: 2024-04-26 04:44:27.354175794 +0000 UTC m=+0.451670034 container init 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 podman[72896]: 2024-04-26 04:44:27.35837561 +0000 UTC m=+0.455869853 container start 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.362Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.362Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.363Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=node_exporter.go:117 level=info collector=arp 2024-04-26T04:44:27.609 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.364Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=edac 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-26T04:44:27.610 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=os 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-26T04:44:27.611 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.365Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=stat 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=time 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=uname 2024-04-26T04:44:27.612 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-26T04:44:27.613 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-26T04:44:27.613 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.366Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-26T04:44:27.613 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.367Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-26T04:44:27.613 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b[72910]: ts=2024-04-26T04:44:27.367Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-26T04:44:27.613 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 bash[72896]: 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 2024-04-26T04:44:27.613 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 04:44:27 smithi055 systemd[1]: Started Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:28 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:28 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:28 smithi055 ceph-mon[30328]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:44:28.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:28 smithi055 ceph-mon[30328]: Reconfiguring daemon grafana.a on smithi055 2024-04-26T04:44:28.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:28 smithi055 ceph-mon[30328]: pgmap v27: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:44:28.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:28 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3858663487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[36743]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[36743]: Reconfiguring daemon grafana.a on smithi055 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[36743]: pgmap v27: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3858663487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:28.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[26454]: Reconfiguring grafana.a (dependencies changed)... 2024-04-26T04:44:28.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[26454]: Reconfiguring daemon grafana.a on smithi055 2024-04-26T04:44:28.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[26454]: pgmap v27: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:44:28.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:28 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3858663487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:31.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:30 smithi038 ceph-mon[36743]: pgmap v28: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-26T04:44:31.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:30 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2306141653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:30 smithi038 ceph-mon[26454]: pgmap v28: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-26T04:44:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2306141653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:30 smithi055 ceph-mon[30328]: pgmap v28: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-04-26T04:44:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2306141653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:33.176 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:32 smithi055 ceph-mon[30328]: pgmap v29: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:44:33.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:32 smithi038 ceph-mon[26454]: pgmap v29: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:44:33.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:32 smithi038 ceph-mon[36743]: pgmap v29: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:44:33.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:33] "GET /metrics HTTP/1.1" 200 36051 "" "Prometheus/2.43.0" 2024-04-26T04:44:34.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:33 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/614364210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:34.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:33 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/614364210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:33 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/614364210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:35.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:34 smithi038 ceph-mon[26454]: pgmap v30: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-26T04:44:35.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:34 smithi038 ceph-mon[36743]: pgmap v30: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-26T04:44:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:34 smithi055 ceph-mon[30328]: pgmap v30: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-26T04:44:36.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:35 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2511385851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:36.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:35 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2511385851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:36.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:35 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2511385851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:37.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:36 smithi038 ceph-mon[26454]: pgmap v31: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-26T04:44:37.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:36 smithi038 ceph-mon[36743]: pgmap v31: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-26T04:44:37.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:36 smithi055 ceph-mon[30328]: pgmap v31: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2024-04-26T04:44:38.916 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:39.207 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:38 smithi055 ceph-mon[30328]: pgmap v32: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:44:39.207 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1596050120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:38 smithi038 ceph-mon[26454]: pgmap v32: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:44:39.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1596050120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:39.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:38 smithi038 ceph-mon[36743]: pgmap v32: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:44:39.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1596050120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:40 smithi055 ceph-mon[30328]: pgmap v33: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-04-26T04:44:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:40 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:40 smithi038 ceph-mon[26454]: pgmap v33: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-04-26T04:44:41.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:40 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:40 smithi038 ceph-mon[36743]: pgmap v33: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2024-04-26T04:44:41.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:40 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:42.223 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:41 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2861801710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:42.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:41 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2861801710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:42.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:41 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2861801710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:42.357 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:42 smithi055 systemd[1]: Stopping Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:42.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:42 smithi055 podman[73189]: 2024-04-26 04:44:42.531720833 +0000 UTC m=+0.157871728 container stop 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, name=ubi8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.openshift.expose-services=) 2024-04-26T04:44:42.857 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:42 smithi055 podman[73189]: 2024-04-26 04:44:42.564469552 +0000 UTC m=+0.190620437 container died 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, distribution-scope=public, version=8.5, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=239.1651231664, name=ubi8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:44:43.176 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:42 smithi055 ceph-mon[30328]: pgmap v34: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2024-04-26T04:44:43.177 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:42 smithi055 podman[73189]: 2024-04-26 04:44:42.985075775 +0000 UTC m=+0.611226658 container cleanup 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.openshift.expose-services=, maintainer=Paul Cuzner , version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.tags=base rhel8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664) 2024-04-26T04:44:43.177 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:42 smithi055 bash[73189]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a 2024-04-26T04:44:43.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:42 smithi038 ceph-mon[26454]: pgmap v34: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2024-04-26T04:44:43.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:42 smithi038 ceph-mon[36743]: pgmap v34: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2024-04-26T04:44:43.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:43] "GET /metrics HTTP/1.1" 200 36043 "" "Prometheus/2.43.0" 2024-04-26T04:44:43.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:43 smithi055 podman[73201]: 2024-04-26 04:44:43.262095593 +0000 UTC m=+0.718348845 container remove 27b61261d3f1b5810a5b96edda512a0298970ab960dde9e59b93fa5429eb94a9 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, vcs-type=git, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, release=239.1651231664) 2024-04-26T04:44:43.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:43 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:43.953 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:43 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2971384504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:43.954 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:43 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:43.954 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:43 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:43.954 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:43 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 6.643s CPU time. 2024-04-26T04:44:43.954 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:43 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:43 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2971384504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:44.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:43 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2971384504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:44.356 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:44 smithi055 podman[73273]: 2024-04-26 04:44:43.972320344 +0000 UTC m=+0.043459226 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:44.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:44 smithi055 podman[73273]: 2024-04-26 04:44:44.500829258 +0000 UTC m=+0.571968132 container create a9a851815363df64cc5d4dfb92adffb40718e434c6479c83da4a5202570c09e1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, version=8.5, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, name=ubi8, vcs-type=git, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:44:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:44 smithi038 ceph-mon[26454]: pgmap v35: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-04-26T04:44:45.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:44 smithi038 ceph-mon[36743]: pgmap v35: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-04-26T04:44:45.251 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:44 smithi055 ceph-mon[30328]: pgmap v35: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2024-04-26T04:44:45.251 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:45 smithi055 podman[73273]: 2024-04-26 04:44:45.250791751 +0000 UTC m=+1.321930632 container init a9a851815363df64cc5d4dfb92adffb40718e434c6479c83da4a5202570c09e1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , description=Ceph Grafana Container, distribution-scope=public, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-26T04:44:45.503 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:45 smithi055 podman[73273]: 2024-04-26 04:44:45.258178687 +0000 UTC m=+1.329317564 container start a9a851815363df64cc5d4dfb92adffb40718e434c6479c83da4a5202570c09e1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, name=ubi8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, vendor=Red Hat, Inc., distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner ) 2024-04-26T04:44:45.503 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:45 smithi055 bash[73273]: a9a851815363df64cc5d4dfb92adffb40718e434c6479c83da4a5202570c09e1 2024-04-26T04:44:45.503 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:45 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:45.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:45] ENGINE Bus STOPPING 2024-04-26T04:44:45.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:45.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:45] ENGINE Bus STOPPED 2024-04-26T04:44:45.858 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:45] ENGINE Bus STARTING 2024-04-26T04:44:45.858 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[73286]: logger=settings t=2024-04-26T04:44:45.614675794Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:45.858 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:45 smithi055 podman[73307]: 2024-04-26 04:44:45.669485819 +0000 UTC m=+0.037392984 container died a9a851815363df64cc5d4dfb92adffb40718e434c6479c83da4a5202570c09e1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vcs-type=git, distribution-scope=public, release=239.1651231664, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vendor=Red Hat, Inc.) 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Serving on http://:::9283 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STARTED 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STOPPING 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STOPPED 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STARTING 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Serving on http://:::9283 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STARTED 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STOPPING 2024-04-26T04:44:46.277 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:44:46.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STOPPED 2024-04-26T04:44:46.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STARTING 2024-04-26T04:44:46.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Serving on http://:::9283 2024-04-26T04:44:46.278 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:44:46] ENGINE Bus STARTED 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055:9095"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055:9095"}]: dispatch 2024-04-26T04:44:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: pgmap v36: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055:3000"}]: dispatch 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055:3000"}]: dispatch 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:46.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:46 smithi055 ceph-mon[30328]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:46.608 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:46 smithi055 podman[73307]: 2024-04-26 04:44:46.298338535 +0000 UTC m=+0.666245676 container remove a9a851815363df64cc5d4dfb92adffb40718e434c6479c83da4a5202570c09e1 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.buildah.version=1.32.0, version=8.5, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=239.1651231664, io.openshift.expose-services=, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8) 2024-04-26T04:44:46.609 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:46 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:44:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055:9095"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055:9095"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: pgmap v36: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:46.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055:3000"}]: dispatch 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055:3000"}]: dispatch 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[26454]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-26T04:44:46.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi038.front.sepia.ceph.com:9093"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055:9095"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055:9095"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:46.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: pgmap v36: 89 pgs: 89 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055:3000"}]: dispatch 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi055:3000"}]: dispatch 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:46.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:46 smithi038 ceph-mon[36743]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-26T04:44:47.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:46 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:44:47.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:46 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.123s CPU time. 2024-04-26T04:44:47.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:47 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3270544768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:47.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:47 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3270544768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:47.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:47 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3270544768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:49.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: pgmap v37: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-26T04:44:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: Upgrade: Updating mgr.y 2024-04-26T04:44:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[36743]: Deploying daemon mgr.y on smithi038 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: pgmap v37: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: Upgrade: Updating mgr.y 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:49.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:49 smithi038 ceph-mon[26454]: Deploying daemon mgr.y on smithi038 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: pgmap v37: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: Upgrade: Updating mgr.y 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:44:49.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:49 smithi055 ceph-mon[30328]: Deploying daemon mgr.y on smithi038 2024-04-26T04:44:50.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:50 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3938127812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:50 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3938127812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:50 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3938127812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:51.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:50 smithi038 systemd[1]: Stopping Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:51 smithi055 ceph-mon[30328]: pgmap v38: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-04-26T04:44:51.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:51 smithi038 ceph-mon[26454]: pgmap v38: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-04-26T04:44:51.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:51 smithi038 ceph-mon[36743]: pgmap v38: 89 pgs: 89 active+clean; 1.8 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-04-26T04:44:52.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:51 smithi038 podman[83145]: 2024-04-26 04:44:51.951127973 +0000 UTC m=+0.726881413 container died 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2) 2024-04-26T04:44:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:52 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4074635919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:52.716 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:52 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4074635919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:52.716 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 podman[83145]: 2024-04-26 04:44:52.429820491 +0000 UTC m=+1.205573932 container cleanup 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS) 2024-04-26T04:44:52.716 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 bash[83145]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y 2024-04-26T04:44:52.717 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:52 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4074635919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:52.967 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 podman[83180]: 2024-04-26 04:44:52.71568415 +0000 UTC m=+0.762672376 container remove 930235385ef781e55478c86f82bcdf5393ccd1915795cf2c2b51dafb5e87f79f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-26T04:44:52.967 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:44:52.967 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-26T04:44:52.967 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 systemd[1]: Stopped Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:52.967 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:52 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 7.600s CPU time. 2024-04-26T04:44:53.574 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:53 smithi038 ceph-mon[26454]: pgmap v39: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:44:53.575 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 systemd[1]: Starting Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:53.575 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:53 smithi038 ceph-mon[36743]: pgmap v39: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:44:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:53 smithi055 ceph-mon[30328]: pgmap v39: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:44:53.607 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:44:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:44:53] "GET /metrics HTTP/1.1" 200 36043 "" "Prometheus/2.43.0" 2024-04-26T04:44:53.888 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 podman[83300]: 2024-04-26 04:44:53.478975265 +0000 UTC m=+0.046452258 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:44:53.889 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 podman[83300]: 2024-04-26 04:44:53.617409522 +0000 UTC m=+0.184886551 container create 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-26T04:44:54.184 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 podman[83300]: 2024-04-26 04:44:53.887550525 +0000 UTC m=+0.455027505 container init 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T04:44:54.185 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 podman[83300]: 2024-04-26 04:44:53.893505541 +0000 UTC m=+0.460982527 container start 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb) 2024-04-26T04:44:54.185 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 bash[83300]: 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 2024-04-26T04:44:54.185 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:53 smithi038 systemd[1]: Started Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:54.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:54.180+0000 7f013176d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:44:54.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:54.541+0000 7f013176d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:44:54.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:54.732+0000 7f013176d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:44:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: pgmap v40: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2024-04-26T04:44:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3274885731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:55 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.454 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:55.316+0000 7f013176d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: pgmap v40: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3274885731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.455 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: pgmap v40: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3274885731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:44:55.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:55.452+0000 7f013176d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:44:57.098 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:56 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-26T04:44:57.098 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:56 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:57.098 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:56 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.123s CPU time. 2024-04-26T04:44:57.098 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:56 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:44:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:57 smithi055 ceph-mon[30328]: pgmap v41: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-04-26T04:44:57.469 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:57 smithi038 ceph-mon[26454]: pgmap v41: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-04-26T04:44:57.470 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:57 smithi038 ceph-mon[36743]: pgmap v41: 89 pgs: 89 active+clean; 1.4 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-04-26T04:44:57.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:57.556+0000 7f013176d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:44:58.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:57 smithi055 podman[73727]: 2024-04-26 04:44:57.696791802 +0000 UTC m=+0.031272259 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:44:58.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:57 smithi055 podman[73727]: 2024-04-26 04:44:57.828383451 +0000 UTC m=+0.162863930 container create f8395f1a1bff726a6815ce662c986daaa12582fd712e1e280359d6709856a7c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vcs-type=git, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, com.redhat.component=ubi8-container) 2024-04-26T04:44:58.131 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:57.858+0000 7f013176d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:44:58.447 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:58 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3752167643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.447 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:58 smithi055 podman[73727]: 2024-04-26 04:44:58.139987045 +0000 UTC m=+0.474467491 container init f8395f1a1bff726a6815ce662c986daaa12582fd712e1e280359d6709856a7c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.buildah.version=1.32.0, name=ubi8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64) 2024-04-26T04:44:58.447 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:58 smithi055 podman[73727]: 2024-04-26 04:44:58.14440414 +0000 UTC m=+0.478884580 container start f8395f1a1bff726a6815ce662c986daaa12582fd712e1e280359d6709856a7c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, release=239.1651231664, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , name=ubi8) 2024-04-26T04:44:58.448 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:58 smithi055 bash[73727]: f8395f1a1bff726a6815ce662c986daaa12582fd712e1e280359d6709856a7c3 2024-04-26T04:44:58.448 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:58 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:44:58.448 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[73756]: logger=settings t=2024-04-26T04:44:58.44622323Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:44:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:58 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3752167643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:58 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:58.154+0000 7f013176d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:44:58.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:58 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3752167643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:44:58.856 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:44:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:44:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:44:58.857 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:44:58 smithi055 podman[73818]: 2024-04-26 04:44:58.499267748 +0000 UTC m=+0.040538210 container died f8395f1a1bff726a6815ce662c986daaa12582fd712e1e280359d6709856a7c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.openshift.expose-services=, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., distribution-scope=public, name=ubi8) 2024-04-26T04:44:58.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:58 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:58.676+0000 7f013176d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:44:58.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:58 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:58.810+0000 7f013176d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:44:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:44:59 smithi038 ceph-mon[26454]: pgmap v42: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-04-26T04:44:59.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:59.137+0000 7f013176d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:44:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:44:59 smithi038 ceph-mon[36743]: pgmap v42: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-04-26T04:44:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:44:59 smithi055 ceph-mon[30328]: pgmap v42: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-04-26T04:44:59.758 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:44:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:44:59.489+0000 7f013176d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:00.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:00.566+0000 7f013176d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:45:01.517 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[26454]: pgmap v43: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-26T04:45:01.517 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2602106842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:01.344+0000 7f013176d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:01.515+0000 7f013176d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[36743]: pgmap v43: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:01.518 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:01.519 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2602106842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:01.586 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:01 smithi055 ceph-mon[30328]: pgmap v43: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-26T04:45:01.586 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:01 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:01.586 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:01 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:01.586 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:01 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2602106842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:01.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:01 smithi055 podman[73818]: 2024-04-26 04:45:01.58593253 +0000 UTC m=+3.127202992 container remove f8395f1a1bff726a6815ce662c986daaa12582fd712e1e280359d6709856a7c3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.openshift.tags=base rhel8, architecture=x86_64, name=ubi8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-26T04:45:01.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:01 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:02.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:01.893+0000 7f013176d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:45:02.606 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:02 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:02.606 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:02 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.076s CPU time. 2024-04-26T04:45:03.227 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:02 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:02.806+0000 7f013176d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:45:03.227 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:02 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:02.974+0000 7f013176d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:45:03.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:03 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:45:03] "GET /metrics HTTP/1.1" 200 36047 "" "Prometheus/2.43.0" 2024-04-26T04:45:03.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:03 smithi055 ceph-mon[30328]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-04-26T04:45:03.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:03 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:03.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:03 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:03.568 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-mon[26454]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-04-26T04:45:03.569 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:03.569 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:03.569 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:03.438+0000 7f013176d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:45:03.569 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-mon[36743]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-04-26T04:45:03.569 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:03.569 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:03.837 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:03.666+0000 7f013176d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:45:04.184 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:03.835+0000 7f013176d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:45:04.184 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:04.025+0000 7f013176d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:04.184 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:04.182+0000 7f013176d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:45:04.436 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2016721077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:04.436 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:04.436 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:04.436 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2016721077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:04.436 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:04.437 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:04 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:04 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2016721077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:04 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:04 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:05 smithi038 ceph-mon[26454]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-04-26T04:45:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:05 smithi038 ceph-mon[36743]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-04-26T04:45:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:05 smithi055 ceph-mon[30328]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 619 op/s 2024-04-26T04:45:05.940 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:05.537+0000 7f013176d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:05.937+0000 7f013176d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:05] ENGINE Bus STARTING 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: CherryPy Checker: 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: The Application mounted at '' has an empty config. 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:06] ENGINE Serving on http://:::9283 2024-04-26T04:45:06.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:06] ENGINE Bus STARTED 2024-04-26T04:45:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: Standby manager daemon y restarted 2024-04-26T04:45:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: Standby manager daemon y started 2024-04-26T04:45:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:45:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:45:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:06 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/504138299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: Standby manager daemon y restarted 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: Standby manager daemon y started 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/504138299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: Standby manager daemon y restarted 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: Standby manager daemon y started 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:06.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:06 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/504138299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:07 smithi055 ceph-mon[30328]: pgmap v46: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-26T04:45:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:07 smithi055 ceph-mon[30328]: mgrmap e26: x(active, since 86s), standbys: y 2024-04-26T04:45:07.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:07 smithi038 ceph-mon[36743]: pgmap v46: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-26T04:45:07.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:07 smithi038 ceph-mon[36743]: mgrmap e26: x(active, since 86s), standbys: y 2024-04-26T04:45:07.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:07 smithi038 ceph-mon[26454]: pgmap v46: 89 pgs: 89 active+clean; 1.0 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-04-26T04:45:07.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:07 smithi038 ceph-mon[26454]: mgrmap e26: x(active, since 86s), standbys: y 2024-04-26T04:45:08.660 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:08.537+0000 7fc498a45700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-26T04:45:08.661 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ignoring --setuser ceph since I am not root 2024-04-26T04:45:08.661 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ignoring --setgroup ceph since I am not root 2024-04-26T04:45:08.883 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:08] ENGINE Bus STOPPING 2024-04-26T04:45:08.884 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:45:08.884 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:08] ENGINE Bus STOPPED 2024-04-26T04:45:08.884 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:08] ENGINE Bus STARTING 2024-04-26T04:45:08.884 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:08] ENGINE Serving on http://:::9283 2024-04-26T04:45:08.884 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:08] ENGINE Bus STARTED 2024-04-26T04:45:08.919 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:08.890+0000 7fa389d9a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:45:08.919 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.948+0000 7f0091b5d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.948+0000 7f0091b5d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.948+0000 7f0091b5d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.948+0000 7f0091b5d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.948+0000 7f0091b5d700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.968+0000 7f008eb57700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.968+0000 7f008eb57700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.968+0000 7f008eb57700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.968+0000 7f008eb57700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:45:08.968+0000 7f008eb57700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:45:09.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:09.224+0000 7fa389d9a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:09.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:09.420+0000 7fa389d9a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: pgmap v47: 89 pgs: 89 active+clean; 873 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: pgmap v48: 89 pgs: 89 active+clean; 873 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:45:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: Activating manager daemon y 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.24941 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: mgrmap e27: y(active, starting, since 0.0158459s) 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:45:09.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: Manager daemon y is now available 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:09.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/578414841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: pgmap v47: 89 pgs: 89 active+clean; 873 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: pgmap v48: 89 pgs: 89 active+clean; 873 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: Activating manager daemon y 2024-04-26T04:45:09.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.24941 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: mgrmap e27: y(active, starting, since 0.0158459s) 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:45:09.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: pgmap v47: 89 pgs: 89 active+clean; 873 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:09.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: pgmap v48: 89 pgs: 89 active+clean; 873 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 172.21.15.55:0/2993553674' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: Activating manager daemon y 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: osdmap e76: 8 total, 8 up, 8 in 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.24941 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: mgrmap e27: y(active, starting, since 0.0158459s) 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:45:09.728 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: Manager daemon y is now available 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:09.729 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/578414841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: Manager daemon y is now available 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:09.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:45:09.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:09.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:45:09.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:09 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/578414841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:10.298 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:09 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:09.997+0000 7fa389d9a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:45:10.298 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:10 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:10.160+0000 7fa389d9a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:45:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:10 smithi055 ceph-mon[30328]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:10 smithi055 ceph-mon[30328]: mgrmap e28: y(active, since 1.0176s) 2024-04-26T04:45:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:10 smithi055 ceph-mon[30328]: pgmap v3: 89 pgs: 89 active+clean; 698 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:45:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:10 smithi038 ceph-mon[26454]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:10 smithi038 ceph-mon[26454]: mgrmap e28: y(active, since 1.0176s) 2024-04-26T04:45:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:10 smithi038 ceph-mon[26454]: pgmap v3: 89 pgs: 89 active+clean; 698 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:45:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:10 smithi038 ceph-mon[36743]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:10 smithi038 ceph-mon[36743]: mgrmap e28: y(active, since 1.0176s) 2024-04-26T04:45:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:10 smithi038 ceph-mon[36743]: pgmap v3: 89 pgs: 89 active+clean; 698 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:45:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:11 smithi055 ceph-mon[30328]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:45:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:11 smithi055 ceph-mon[30328]: Cluster is now healthy 2024-04-26T04:45:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:11 smithi055 ceph-mon[30328]: mgrmap e29: y(active, since 2s) 2024-04-26T04:45:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:11 smithi055 ceph-mon[30328]: [26/Apr/2024:04:45:10] ENGINE Bus STARTING 2024-04-26T04:45:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:11 smithi055 ceph-mon[30328]: [26/Apr/2024:04:45:10] ENGINE Serving on http://172.21.15.38:8765 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[26454]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[26454]: Cluster is now healthy 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[26454]: mgrmap e29: y(active, since 2s) 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[26454]: [26/Apr/2024:04:45:10] ENGINE Bus STARTING 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[26454]: [26/Apr/2024:04:45:10] ENGINE Serving on http://172.21.15.38:8765 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[36743]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[36743]: Cluster is now healthy 2024-04-26T04:45:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[36743]: mgrmap e29: y(active, since 2s) 2024-04-26T04:45:11.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[36743]: [26/Apr/2024:04:45:10] ENGINE Bus STARTING 2024-04-26T04:45:11.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:11 smithi038 ceph-mon[36743]: [26/Apr/2024:04:45:10] ENGINE Serving on http://172.21.15.38:8765 2024-04-26T04:45:12.540 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:12.240+0000 7fa389d9a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:45:12.540 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:12 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-26T04:45:12.540 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:12 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:12.541 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:12 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.076s CPU time. 2024-04-26T04:45:12.541 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:12 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:12.799 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:12 smithi055 ceph-mon[30328]: [26/Apr/2024:04:45:11] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:45:12.799 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:12 smithi055 ceph-mon[30328]: [26/Apr/2024:04:45:11] ENGINE Bus STARTED 2024-04-26T04:45:12.799 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:12 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3706052192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:12.799 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:12.538+0000 7fa389d9a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:12.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:12 smithi038 ceph-mon[26454]: [26/Apr/2024:04:45:11] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:45:12.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:12 smithi038 ceph-mon[26454]: [26/Apr/2024:04:45:11] ENGINE Bus STARTED 2024-04-26T04:45:12.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:12 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3706052192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:12.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:12 smithi038 ceph-mon[36743]: [26/Apr/2024:04:45:11] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:45:12.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:12 smithi038 ceph-mon[36743]: [26/Apr/2024:04:45:11] ENGINE Bus STARTED 2024-04-26T04:45:12.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:12 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3706052192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:13.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:12.837+0000 7fa389d9a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:45:13.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:12 smithi055 podman[74490]: 2024-04-26 04:45:12.701125067 +0000 UTC m=+0.057992492 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:13.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:12 smithi055 podman[74490]: 2024-04-26 04:45:12.822058038 +0000 UTC m=+0.178925454 container create bd119ac009d15211c99e0115e49d6fdc98ea27ae84733896a3453935cff3f4d0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.buildah.version=1.32.0, release=239.1651231664, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, description=Ceph Grafana Container, com.redhat.component=ubi8-container) 2024-04-26T04:45:13.496 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:13.380+0000 7fa389d9a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:45:13.496 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:13 smithi055 podman[74490]: 2024-04-26 04:45:13.136819817 +0000 UTC m=+0.493687240 container init bd119ac009d15211c99e0115e49d6fdc98ea27ae84733896a3453935cff3f4d0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., release=239.1651231664, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, version=8.5, architecture=x86_64, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:45:13.496 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:13 smithi055 podman[74490]: 2024-04-26 04:45:13.141734252 +0000 UTC m=+0.498601662 container start bd119ac009d15211c99e0115e49d6fdc98ea27ae84733896a3453935cff3f4d0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, version=8.5, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, distribution-scope=public, io.openshift.expose-services=, name=ubi8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664) 2024-04-26T04:45:13.496 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:13 smithi055 bash[74490]: bd119ac009d15211c99e0115e49d6fdc98ea27ae84733896a3453935cff3f4d0 2024-04-26T04:45:13.496 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:13 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:13.755 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:13 smithi055 ceph-mon[30328]: pgmap v5: 89 pgs: 89 active+clean; 698 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:45:13.755 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:13 smithi055 ceph-mon[30328]: mgrmap e30: y(active, since 4s) 2024-04-26T04:45:13.755 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:13.559+0000 7fa389d9a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:45:13.755 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[74520]: logger=settings t=2024-04-26T04:45:13.49578315Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:13.755 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:13 smithi055 podman[74580]: 2024-04-26 04:45:13.74325947 +0000 UTC m=+0.220889665 container died bd119ac009d15211c99e0115e49d6fdc98ea27ae84733896a3453935cff3f4d0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, version=8.5, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, name=ubi8, com.redhat.component=ubi8-container, vcs-type=git, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.tags=base rhel8, architecture=x86_64, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:45:13.838 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:13 smithi038 ceph-mon[26454]: pgmap v5: 89 pgs: 89 active+clean; 698 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:45:13.838 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:13 smithi038 ceph-mon[26454]: mgrmap e30: y(active, since 4s) 2024-04-26T04:45:13.838 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:13 smithi038 ceph-mon[36743]: pgmap v5: 89 pgs: 89 active+clean; 698 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:45:13.838 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:13 smithi038 ceph-mon[36743]: mgrmap e30: y(active, since 4s) 2024-04-26T04:45:14.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:13.893+0000 7fa389d9a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:45:14.596 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:14 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:14.230+0000 7fa389d9a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:14 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1502683673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:14 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1502683673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:14.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:14 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1502683673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:15.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:15.247+0000 7fa389d9a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:45:15.357 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:15 smithi055 podman[74580]: 2024-04-26 04:45:15.071065501 +0000 UTC m=+1.548695696 container remove bd119ac009d15211c99e0115e49d6fdc98ea27ae84733896a3453935cff3f4d0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, name=ubi8, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, com.redhat.component=ubi8-container, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vendor=Red Hat, Inc.) 2024-04-26T04:45:15.357 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:15 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:15.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:15 smithi055 ceph-mon[30328]: pgmap v6: 89 pgs: 89 active+clean; 687 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s 2024-04-26T04:45:15.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:15 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:15.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:15 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:15 smithi038 ceph-mon[26454]: pgmap v6: 89 pgs: 89 active+clean; 687 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s 2024-04-26T04:45:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:15 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:15 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:15.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:15 smithi038 ceph-mon[36743]: pgmap v6: 89 pgs: 89 active+clean; 687 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 36 op/s 2024-04-26T04:45:15.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:15 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:15.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:15 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:16.270 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:16.000+0000 7fa389d9a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:45:16.271 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:16.173+0000 7fa389d9a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:45:16.539 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:16 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:16.540 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:16 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.175s CPU time. 2024-04-26T04:45:16.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:16 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:16.536+0000 7fa389d9a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:45:17.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-mon[30328]: pgmap v7: 89 pgs: 89 active+clean; 490 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 494 op/s 2024-04-26T04:45:17.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3711947786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:17.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:17.456+0000 7fa389d9a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:45:17.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:17 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:17.623+0000 7fa389d9a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:45:17.908 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[26454]: pgmap v7: 89 pgs: 89 active+clean; 490 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 494 op/s 2024-04-26T04:45:17.908 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3711947786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[36743]: pgmap v7: 89 pgs: 89 active+clean; 490 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 494 op/s 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:17.909 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:17 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3711947786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:18.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:18.053+0000 7fa389d9a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:45:18.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:18.288+0000 7fa389d9a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:45:18.837 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:18.467+0000 7fa389d9a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:45:18.838 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:18.676+0000 7fa389d9a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:19.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:18.836+0000 7fa389d9a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:45:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:19 smithi038 ceph-mon[26454]: pgmap v8: 89 pgs: 89 active+clean; 490 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 384 op/s 2024-04-26T04:45:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:19 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:19 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:19 smithi038 ceph-mon[36743]: pgmap v8: 89 pgs: 89 active+clean; 490 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 384 op/s 2024-04-26T04:45:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:19 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:19 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:19 smithi055 ceph-mon[30328]: pgmap v8: 89 pgs: 89 active+clean; 490 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 384 op/s 2024-04-26T04:45:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:19 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:19 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:20.514 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:20.214+0000 7fa389d9a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:45:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1930960696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-mon[30328]: Standby manager daemon x started 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:20.582+0000 7fa389d9a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:45:20] ENGINE Bus STARTING 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: CherryPy Checker: 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: The Application mounted at '' has an empty config. 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: 2024-04-26T04:45:20.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:45:20] ENGINE Serving on http://:::9283 2024-04-26T04:45:20.858 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: [26/Apr/2024:04:45:20] ENGINE Bus STARTED 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1930960696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[26454]: Standby manager daemon x started 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1930960696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:20.893 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[36743]: Standby manager daemon x started 2024-04-26T04:45:20.894 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:20.894 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:20.894 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:20.894 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:20 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2392472077' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:21 smithi038 ceph-mon[26454]: pgmap v9: 89 pgs: 89 active+clean; 271 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 634 op/s 2024-04-26T04:45:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:21 smithi038 ceph-mon[26454]: mgrmap e31: y(active, since 12s), standbys: x 2024-04-26T04:45:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:21 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:45:21.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:21 smithi038 ceph-mon[36743]: pgmap v9: 89 pgs: 89 active+clean; 271 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 634 op/s 2024-04-26T04:45:21.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:21 smithi038 ceph-mon[36743]: mgrmap e31: y(active, since 12s), standbys: x 2024-04-26T04:45:21.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:21 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:45:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:21 smithi055 ceph-mon[30328]: pgmap v9: 89 pgs: 89 active+clean; 271 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 634 op/s 2024-04-26T04:45:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:21 smithi055 ceph-mon[30328]: mgrmap e31: y(active, since 12s), standbys: x 2024-04-26T04:45:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:21 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:45:22.968 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:22 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3580630151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:22.968 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:22 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3580630151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:22 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3580630151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:23.610 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:23 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[64364]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:45:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-26T04:45:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[26454]: pgmap v10: 89 pgs: 89 active+clean; 271 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 579 op/s 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[36743]: pgmap v10: 89 pgs: 89 active+clean; 271 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 579 op/s 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:23.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:23 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:23 smithi055 ceph-mon[30328]: pgmap v10: 89 pgs: 89 active+clean; 271 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 579 op/s 2024-04-26T04:45:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:23 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:23 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:23 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:23 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:24.674 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:24.370Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-04-26T04:45:24.674 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:24.374Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-04-26T04:45:24.674 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:24.374Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-04-26T04:45:24.674 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:24.378Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-04-26T04:45:24.674 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:24.378Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.55:8765: connect: connection refused" 2024-04-26T04:45:24.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:45:24.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:45:24.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:24.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:24.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:24.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:45:24.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:45:24.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:24.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:24.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:24 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:45:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:45:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:24 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:45:25.121 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Removed 133496 objects 2024-04-26T04:45:25.121 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Clean up completed and total clean up time :169.079 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: pgmap v11: 89 pgs: 89 active+clean; 255 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 600 op/s 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1846096755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: pgmap v11: 89 pgs: 89 active+clean; 255 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 600 op/s 2024-04-26T04:45:25.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:25.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:25 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1846096755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:26.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:26.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:45:26.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: pgmap v11: 89 pgs: 89 active+clean; 255 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 600 op/s 2024-04-26T04:45:26.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:26.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:45:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:26.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:25 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1846096755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:26.434 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:26 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-26T04:45:26.435 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:26 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:26.435 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:26 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.175s CPU time. 2024-04-26T04:45:26.435 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:26 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:26.748 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:26 smithi055 ceph-mon[30328]: pgmap v12: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 1.01k op/s 2024-04-26T04:45:26.748 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:26 smithi055 ceph-mon[30328]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:45:26.748 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:26 smithi055 ceph-mon[30328]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:45:26.749 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:26 smithi055 ceph-mon[30328]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:26.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[26454]: pgmap v12: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 1.01k op/s 2024-04-26T04:45:26.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[26454]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:45:26.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[26454]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:45:26.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[26454]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:26.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[36743]: pgmap v12: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 1.01k op/s 2024-04-26T04:45:26.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[36743]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-26T04:45:26.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[36743]: Reconfiguring daemon prometheus.a on smithi055 2024-04-26T04:45:26.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:26 smithi038 ceph-mon[36743]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:45:27.805 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:27 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1837494042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:27.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:27 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1837494042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:27.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:27 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1837494042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:28.348 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-04-26T04:45:28.348 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-04-26T04:45:28.628 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:28.883 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:28 smithi055 ceph-mon[30328]: pgmap v13: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-04-26T04:45:28.884 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:28 smithi055 podman[76040]: 2024-04-26 04:45:28.510704368 +0000 UTC m=+0.022085625 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:28.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:28 smithi038 ceph-mon[26454]: pgmap v13: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-04-26T04:45:28.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:28 smithi038 ceph-mon[36743]: pgmap v13: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-04-26T04:45:29.246 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 systemd[1]: Stopping Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:29.246 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 podman[76040]: 2024-04-26 04:45:29.130120181 +0000 UTC m=+0.641501426 container create 751c31c17cd9de904c08721a4d391bf8070f50c2840a026d3ec859e9b5fcdf92 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , name=ubi8, io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., distribution-scope=public, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container) 2024-04-26T04:45:29.510 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 podman[76040]: 2024-04-26 04:45:29.349379248 +0000 UTC m=+0.860760492 container init 751c31c17cd9de904c08721a4d391bf8070f50c2840a026d3ec859e9b5fcdf92 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, vcs-type=git, vendor=Red Hat, Inc., maintainer=Paul Cuzner , version=8.5, io.openshift.expose-services=, release=239.1651231664) 2024-04-26T04:45:29.510 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 podman[76040]: 2024-04-26 04:45:29.352170837 +0000 UTC m=+0.863552082 container start 751c31c17cd9de904c08721a4d391bf8070f50c2840a026d3ec859e9b5fcdf92 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, name=ubi8, description=Ceph Grafana Container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., release=239.1651231664, version=8.5, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.component=ubi8-container) 2024-04-26T04:45:29.511 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 bash[76040]: 751c31c17cd9de904c08721a4d391bf8070f50c2840a026d3ec859e9b5fcdf92 2024-04-26T04:45:29.511 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:29.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[76087]: logger=settings t=2024-04-26T04:45:29.509668569Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:29.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:29 smithi055 podman[76122]: 2024-04-26 04:45:29.572995013 +0000 UTC m=+0.055969537 container died 751c31c17cd9de904c08721a4d391bf8070f50c2840a026d3ec859e9b5fcdf92 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, distribution-scope=public, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-type=git, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[72677]: ts=2024-04-26T04:45:29.561Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T04:45:29.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:29 smithi055 podman[76097]: 2024-04-26 04:45:29.571961327 +0000 UTC m=+0.306377397 container died 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:30.293 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:30 smithi055 podman[76097]: 2024-04-26 04:45:30.086560254 +0000 UTC m=+0.820976305 container cleanup 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:30.293 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:30 smithi055 bash[76097]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T04:45:30.718 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_0' removed 2024-04-26T04:45:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:30 smithi055 ceph-mon[30328]: pgmap v14: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-04-26T04:45:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/4098407410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:30 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/109851428' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:45:30.857 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:30 smithi055 podman[76122]: 2024-04-26 04:45:30.665297325 +0000 UTC m=+1.148271834 container remove 751c31c17cd9de904c08721a4d391bf8070f50c2840a026d3ec859e9b5fcdf92 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, name=ubi8, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , vcs-type=git, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, release=239.1651231664) 2024-04-26T04:45:30.857 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:30 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:31.064 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:30 smithi038 ceph-mon[26454]: pgmap v14: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-04-26T04:45:31.064 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/4098407410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:31.064 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:30 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/109851428' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:45:31.064 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:30 smithi038 ceph-mon[36743]: pgmap v14: 89 pgs: 89 active+clean; 24 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-04-26T04:45:31.064 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:30 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/4098407410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:31.064 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:30 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/109851428' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:45:31.310 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 podman[76134]: 2024-04-26 04:45:31.117575142 +0000 UTC m=+1.547135190 container remove 1c33f7edec9e1d083790e726b490eaefc2570aee554c61a4a0470ffc41423b94 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:31.441 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-26T04:45:31.441 INFO:tasks.radosbench:Beginning radosbench... 2024-04-26T04:45:31.441 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-04-26T04:45:31.442 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-04-26T04:45:31.606 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:31 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:31.606 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-26T04:45:31.606 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 systemd[1]: Stopped Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:31.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.397s CPU time. 2024-04-26T04:45:31.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 systemd[1]: Starting Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:31.708 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:31.996 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:31 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/109851428' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:45:31.996 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:31 smithi055 ceph-mon[30328]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:45:31.996 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 podman[76271]: 2024-04-26 04:45:31.529919822 +0000 UTC m=+0.019842420 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-26T04:45:31.996 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 podman[76271]: 2024-04-26 04:45:31.652197006 +0000 UTC m=+0.142119593 container create c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:32.059 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:31 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/109851428' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:45:32.059 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:31 smithi038 ceph-mon[26454]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:45:32.059 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:31 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/109851428' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:45:32.060 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:31 smithi038 ceph-mon[36743]: osdmap e77: 8 total, 8 up, 8 in 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 podman[76271]: 2024-04-26 04:45:31.996024779 +0000 UTC m=+0.485947365 container init c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:31 smithi055 podman[76271]: 2024-04-26 04:45:31.99891233 +0000 UTC m=+0.488834916 container start c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.024Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.024Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.024Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi055 (none))" 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.024Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.024Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-26T04:45:32.357 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.026Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.026Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.027Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.027Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.029Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.029Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.696µs 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.029Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.030Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.046Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.048Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.049Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.049Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.029µs wal_replay_duration=19.269638ms wbl_replay_duration=133ns total_replay_duration=19.309973ms 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.052Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-26T04:45:32.358 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.052Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-26T04:45:32.359 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.052Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-26T04:45:32.359 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.068Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.210064ms db_storage=1.089µs remote_storage=1.194µs web_handler=353ns query_engine=788ns scrape=399.912µs scrape_sd=86.971µs notify=17.877µs notify_sd=24.431µs rules=14.441931ms tracing=4.081µs 2024-04-26T04:45:32.359 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.068Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-26T04:45:32.359 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:32.068Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-26T04:45:32.359 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 bash[76271]: c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 2024-04-26T04:45:32.359 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:32 smithi055 systemd[1]: Started Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:32] ENGINE Bus STOPPING 2024-04-26T04:45:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T04:45:32.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:32] ENGINE Bus STOPPED 2024-04-26T04:45:32.476 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:32] ENGINE Bus STARTING 2024-04-26T04:45:32.476 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:32] ENGINE Serving on http://:::9283 2024-04-26T04:45:32.476 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:45:32] ENGINE Bus STARTED 2024-04-26T04:45:32.873 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.873 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:32.873 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:32.873 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:32.873 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:32.873 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2411338867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:32.874 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:32 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2411338867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:33.100 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: pgmap v16: 73 pgs: 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:33.100 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:33.100 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:33.100 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:33.101 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:33.101 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:33.101 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:33.101 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:32 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2411338867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:34.129 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_1' created 2024-04-26T04:45:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:34 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:34 smithi055 ceph-mon[30328]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:34 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:34 smithi038 ceph-mon[26454]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:34 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-26T04:45:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:34 smithi038 ceph-mon[36743]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi055.front.sepia.ceph.com:9095"}]: dispatch 2024-04-26T04:45:34.856 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-04-26T04:45:35.081 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:35.081 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3994787848' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-26T04:45:35.081 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: Upgrade: Updating mgr.x 2024-04-26T04:45:35.081 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:45:35.081 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:45:35.081 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: Deploying daemon mgr.x on smithi055 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3994787848' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2453664023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3994787848' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: Upgrade: Updating mgr.x 2024-04-26T04:45:35.082 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: Deploying daemon mgr.x on smithi055 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3994787848' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:45:35.083 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:35 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2453664023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:35.123 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:45:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: pgmap v17: 73 pgs: 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3994787848' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-26T04:45:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: Upgrade: Updating mgr.x 2024-04-26T04:45:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: Deploying daemon mgr.x on smithi055 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3994787848' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: osdmap e78: 8 total, 8 up, 8 in 2024-04-26T04:45:35.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:35 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2453664023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:36.322 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:36 smithi055 ceph-mon[30328]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:45:36.322 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:36 smithi055 ceph-mon[30328]: pgmap v20: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.323 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 systemd[1]: Stopping Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:36.373 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:36 smithi038 ceph-mon[36743]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:45:36.373 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:36 smithi038 ceph-mon[36743]: pgmap v20: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.373 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:36 smithi038 ceph-mon[26454]: osdmap e79: 8 total, 8 up, 8 in 2024-04-26T04:45:36.373 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:36 smithi038 ceph-mon[26454]: pgmap v20: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:36.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 podman[76611]: 2024-04-26 04:45:36.321413694 +0000 UTC m=+0.117411980 container stop f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-26T04:45:36.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 podman[76611]: 2024-04-26 04:45:36.351034644 +0000 UTC m=+0.147032936 container died f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:45:36.993 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 podman[76611]: 2024-04-26 04:45:36.766580315 +0000 UTC m=+0.562578590 container cleanup f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True) 2024-04-26T04:45:36.994 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 bash[76611]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x 2024-04-26T04:45:37.147 INFO:teuthology.orchestra.run.smithi038.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-04-26T04:45:37.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:37 smithi055 ceph-mon[30328]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:45:37.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:37 smithi055 ceph-mon[30328]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:45:37.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:37 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/551695076' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:45:37.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 podman[76626]: 2024-04-26 04:45:36.993509547 +0000 UTC m=+0.666345690 container remove f20aca4e8d349a4845b991c5f45c3317a6436ea47522b533d2f78c0469328ddd (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:45:37.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:36 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-26T04:45:37.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-26T04:45:37.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 systemd[1]: Stopped Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:37.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service: Consumed 43.177s CPU time. 2024-04-26T04:45:37.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:37 smithi038 ceph-mon[26454]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:45:37.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:37 smithi038 ceph-mon[26454]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:45:37.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:37 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/551695076' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:45:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:37 smithi038 ceph-mon[36743]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:45:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:37 smithi038 ceph-mon[36743]: osdmap e80: 8 total, 8 up, 8 in 2024-04-26T04:45:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:37 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/551695076' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:45:37.642 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 systemd[1]: Starting Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:37.950 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 podman[76756]: 2024-04-26 04:45:37.544874586 +0000 UTC m=+0.018736859 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:45:37.950 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 podman[76756]: 2024-04-26 04:45:37.645069094 +0000 UTC m=+0.118931388 container create f26c41ca13263a58820f6d2481dddf1ea491be337c627a7b22a2a03abd99520f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb) 2024-04-26T04:45:37.951 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 podman[76756]: 2024-04-26 04:45:37.855970179 +0000 UTC m=+0.329832431 container init f26c41ca13263a58820f6d2481dddf1ea491be337c627a7b22a2a03abd99520f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb) 2024-04-26T04:45:37.951 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 podman[76756]: 2024-04-26 04:45:37.858690054 +0000 UTC m=+0.332552308 container start f26c41ca13263a58820f6d2481dddf1ea491be337c627a7b22a2a03abd99520f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T04:45:37.951 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 ceph-mgr[76776]: -- 172.21.15.55:0/2167370043 <== mon.1 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b88dca5a0 con 0x555b88d9ec00 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 bash[76756]: f26c41ca13263a58820f6d2481dddf1ea491be337c627a7b22a2a03abd99520f 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:37 smithi055 systemd[1]: Started Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:38.015+0000 7fe60e53c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:38.215+0000 7fe60e53c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/551695076' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: pgmap v23: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3136427481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:38.217 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:38.218 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:38.218 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/551695076' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: pgmap v23: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/3136427481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/551695076' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: pgmap v23: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3136427481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:38 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:38.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:38.324+0000 7fe60e53c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:45:38.893 DEBUG:teuthology.orchestra.run.smithi038:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-04-26T04:45:38.896 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-26T04:45:38.896 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-26T04:45:38.949 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:hints = 1 2024-04-26T04:45:38.949 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-26T04:45:38.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object prefix: benchmark_data_smithi038_92445 2024-04-26T04:45:38.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:45:38.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 0 0 0 0 0 0 - 0 2024-04-26T04:45:39.059 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:38.700+0000 7fe60e53c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:45:39.060 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:38.791+0000 7fe60e53c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:45:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:39 smithi055 ceph-mon[30328]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:45:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:39 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:39 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:39 smithi038 ceph-mon[36743]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:45:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:39 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:39 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:39.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:39 smithi038 ceph-mon[26454]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:45:39.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:39 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:39.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:39 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:39.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 1 16 3543 3527 220.425 220.438 0.00631241 0.00451707 2024-04-26T04:45:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:40 smithi038 ceph-mon[26454]: pgmap v24: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:40 smithi038 ceph-mon[36743]: pgmap v24: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:40 smithi055 ceph-mon[30328]: pgmap v24: 89 pgs: 16 creating+peering, 73 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:45:40.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 2 16 6914 6898 215.547 210.688 0.00400795 0.00462954 2024-04-26T04:45:41.224 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:41 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/1197071245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:41.224 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:40.943+0000 7fe60e53c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:45:41.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:41 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/1197071245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:41.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:41 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/1197071245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:41.596 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:41.288+0000 7fe60e53c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:41.596 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:41.593+0000 7fe60e53c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:45:41.597 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:41 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-26T04:45:41.597 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:41 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:41.597 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:41 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:41.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 3 16 10200 10184 212.15 205.375 0.00236665 0.00470636 2024-04-26T04:45:42.329 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:42 smithi055 ceph-mon[30328]: pgmap v25: 89 pgs: 89 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 381 op/s 2024-04-26T04:45:42.329 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:42.161+0000 7fe60e53c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:45:42.329 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:42.327+0000 7fe60e53c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:45:42.329 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:42 smithi055 podman[77267]: 2024-04-26 04:45:42.078642695 +0000 UTC m=+0.364408561 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:42 smithi038 ceph-mon[26454]: pgmap v25: 89 pgs: 89 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 381 op/s 2024-04-26T04:45:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:42 smithi038 ceph-mon[36743]: pgmap v25: 89 pgs: 89 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 381 op/s 2024-04-26T04:45:42.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:42.651+0000 7fe60e53c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:45:42.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:42 smithi055 podman[77267]: 2024-04-26 04:45:42.597077031 +0000 UTC m=+0.882842881 container create 2d3a8e32d9446047381c2f3015cd39f5280254a4102ad82e5280694a357e8ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, distribution-scope=public, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, name=ubi8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5) 2024-04-26T04:45:42.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 4 16 13136 13120 204.984 183.5 0.00598297 0.00487128 2024-04-26T04:45:43.270 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:43 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2973384148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:43.270 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:42.999+0000 7fe60e53c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:45:43.270 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:43 smithi055 podman[77267]: 2024-04-26 04:45:43.158354507 +0000 UTC m=+1.444120355 container init 2d3a8e32d9446047381c2f3015cd39f5280254a4102ad82e5280694a357e8ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, release=239.1651231664, version=8.5, io.openshift.expose-services=, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, architecture=x86_64, io.openshift.tags=base rhel8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, name=ubi8, maintainer=Paul Cuzner , vendor=Red Hat, Inc.) 2024-04-26T04:45:43.270 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:43 smithi055 podman[77267]: 2024-04-26 04:45:43.169214419 +0000 UTC m=+1.454980266 container start 2d3a8e32d9446047381c2f3015cd39f5280254a4102ad82e5280694a357e8ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.buildah.version=1.32.0, version=8.5, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.openshift.expose-services=, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:45:43.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:43 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2973384148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:43.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:43 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2973384148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:43.605 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:43 smithi055 bash[77267]: 2d3a8e32d9446047381c2f3015cd39f5280254a4102ad82e5280694a357e8ba7 2024-04-26T04:45:43.605 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:43 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:43.605 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[77327]: logger=settings t=2024-04-26T04:45:43.504535705Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:43.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 5 16 15913 15897 198.697 173.562 0.00549773 0.00502556 2024-04-26T04:45:44.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:44 smithi055 ceph-mon[30328]: pgmap v26: 89 pgs: 89 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s 2024-04-26T04:45:44.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:44.083+0000 7fe60e53c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:45:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:44 smithi038 ceph-mon[26454]: pgmap v26: 89 pgs: 89 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s 2024-04-26T04:45:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:44 smithi038 ceph-mon[36743]: pgmap v26: 89 pgs: 89 active+clean; 145 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s wr, 288 op/s 2024-04-26T04:45:44.950 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 6 16 18760 18744 195.234 177.938 0.0046867 0.00511424 2024-04-26T04:45:45.084 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:44.882+0000 7fe60e53c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:45:45.084 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:44 smithi055 podman[77367]: 2024-04-26 04:45:44.818346013 +0000 UTC m=+1.300021174 container remove 2d3a8e32d9446047381c2f3015cd39f5280254a4102ad82e5280694a357e8ba7 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner ) 2024-04-26T04:45:45.084 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:44 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:45.356 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:45.083+0000 7fe60e53c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:45:45.356 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:45 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:45.357 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:45 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.137s CPU time. 2024-04-26T04:45:45.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:45.475+0000 7fe60e53c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:45:45.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 7 16 21163 21147 188.797 150.188 0.00869343 0.0052906 2024-04-26T04:45:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:45 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:45 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:45 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2485528691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:46.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:45 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:46.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:45 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:46.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:45 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2485528691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:46.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:45 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:46.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:45 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:46.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:45 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2485528691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:46.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:46.434+0000 7fe60e53c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:45:46.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:46.615+0000 7fe60e53c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:45:46.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 8 15 23661 23646 184.719 156.188 0.00562162 0.0054077 2024-04-26T04:45:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-mon[26454]: pgmap v27: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-04-26T04:45:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:45:46] "GET /metrics HTTP/1.1" 200 36066 "" "Prometheus/2.43.0" 2024-04-26T04:45:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-mon[36743]: pgmap v27: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-04-26T04:45:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:47.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:46 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:47.333 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:47.080+0000 7fe60e53c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:45:47.333 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:46 smithi055 ceph-mon[30328]: pgmap v27: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-04-26T04:45:47.333 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:46 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:47.333 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:46 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:47.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:47.329+0000 7fe60e53c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:45:47.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:47.500+0000 7fe60e53c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:45:47.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 9 16 25974 25958 180.249 144.5 0.00528405 0.00554207 2024-04-26T04:45:47.999 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:47.714+0000 7fe60e53c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:45:48.000 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:47.881+0000 7fe60e53c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:45:48.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 10 16 27904 27888 174.285 120.625 0.00502695 0.00573278 2024-04-26T04:45:48.975 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:48 smithi055 ceph-mon[30328]: pgmap v28: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-04-26T04:45:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:48 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2527468156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:48 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:49.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:49.181+0000 7fe60e53c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:45:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:48 smithi038 ceph-mon[26454]: pgmap v28: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-04-26T04:45:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:48 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2527468156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:48 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:48 smithi038 ceph-mon[36743]: pgmap v28: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-04-26T04:45:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:48 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2527468156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:48 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:49.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:49.527+0000 7fe60e53c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:45:49.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: [26/Apr/2024:04:45:49] ENGINE Bus STARTING 2024-04-26T04:45:49.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: CherryPy Checker: 2024-04-26T04:45:49.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: The Application mounted at '' has an empty config. 2024-04-26T04:45:49.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:45:49.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: [26/Apr/2024:04:45:49] ENGINE Serving on http://:::9283 2024-04-26T04:45:49.857 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: [26/Apr/2024:04:45:49] ENGINE Bus STARTED 2024-04-26T04:45:49.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 11 16 29839 29823 169.434 120.938 0.00635683 0.00589625 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: pgmap v29: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: Standby manager daemon x restarted 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: Standby manager daemon x started 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:50.275 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:49 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: pgmap v29: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-04-26T04:45:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: Standby manager daemon x restarted 2024-04-26T04:45:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: Standby manager daemon x started 2024-04-26T04:45:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[26454]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: pgmap v29: 89 pgs: 89 active+clean; 1.1 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: Standby manager daemon x restarted 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: Standby manager daemon x started 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:45:50.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:49 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/2315853905' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:45:50.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 12 16 31801 31785 165.533 122.625 0.00844122 0.00603456 2024-04-26T04:45:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:50 smithi038 ceph-mon[26454]: mgrmap e32: y(active, since 41s), standbys: x 2024-04-26T04:45:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:50 smithi038 ceph-mon[36743]: mgrmap e32: y(active, since 41s), standbys: x 2024-04-26T04:45:51.272 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:50 smithi055 ceph-mon[30328]: mgrmap e32: y(active, since 41s), standbys: x 2024-04-26T04:45:51.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 13 16 33687 33671 161.866 117.875 0.00377878 0.00617212 2024-04-26T04:45:52.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:52 smithi055 ceph-mon[30328]: pgmap v30: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-26T04:45:52.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:52 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/363265644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:52 smithi038 ceph-mon[26454]: pgmap v30: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-26T04:45:52.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:52 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/363265644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:52 smithi038 ceph-mon[36743]: pgmap v30: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-04-26T04:45:52.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:52 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/363265644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:52.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 14 16 35859 35843 159.999 135.75 0.00601725 0.00624422 2024-04-26T04:45:53.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 15 16 37874 37858 157.728 125.938 0.00536926 0.00633408 2024-04-26T04:45:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:54 smithi055 ceph-mon[30328]: pgmap v31: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:45:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:54 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:54 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2654188527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:54.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:54 smithi038 ceph-mon[36743]: pgmap v31: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:45:54.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:54 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:54.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:54 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2654188527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:54.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:54 smithi038 ceph-mon[26454]: pgmap v31: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2024-04-26T04:45:54.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:54 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:45:54.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:54 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2654188527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:54.951 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 16 16 39772 39756 155.283 118.625 0.0034835 0.00643365 2024-04-26T04:45:55.603 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.603 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.603 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:55.603 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: pgmap v32: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: pgmap v33: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: Upgrade: Setting container_image for all mgr 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:45:55.604 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[26454]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: pgmap v32: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: pgmap v33: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: Upgrade: Setting container_image for all mgr 2024-04-26T04:45:55.605 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:45:55.606 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:55 smithi038 ceph-mon[36743]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:45:55.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-26T04:45:55.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:55.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.137s CPU time. 2024-04-26T04:45:55.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: pgmap v32: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: pgmap v33: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: Upgrade: Setting container_image for all mgr 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-26T04:45:55.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:55 smithi055 ceph-mon[30328]: Upgrade: It appears safe to stop mon.a 2024-04-26T04:45:55.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 17 16 41606 41590 152.891 114.625 0.00913144 0.00653493 2024-04-26T04:45:55.953 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 podman[80271]: 2024-04-26 04:45:55.564963692 +0000 UTC m=+0.033965561 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:45:55.953 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 podman[80271]: 2024-04-26 04:45:55.691268093 +0000 UTC m=+0.160269936 container create 842bb596357f6c8ec0beb9edd3c245ccc4d04942d32f3210a4a0d6b8fb81fa0e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, vcs-type=git, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.component=ubi8-container, release=239.1651231664, name=ubi8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:45:56.230 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 podman[80271]: 2024-04-26 04:45:55.953014077 +0000 UTC m=+0.422015930 container init 842bb596357f6c8ec0beb9edd3c245ccc4d04942d32f3210a4a0d6b8fb81fa0e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, release=239.1651231664, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.buildah.version=1.32.0, distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:45:56.230 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:55 smithi055 podman[80271]: 2024-04-26 04:45:55.957465938 +0000 UTC m=+0.426467783 container start 842bb596357f6c8ec0beb9edd3c245ccc4d04942d32f3210a4a0d6b8fb81fa0e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., io.buildah.version=1.32.0, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, release=239.1651231664, name=ubi8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:45:56.230 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:56 smithi055 bash[80271]: 842bb596357f6c8ec0beb9edd3c245ccc4d04942d32f3210a4a0d6b8fb81fa0e 2024-04-26T04:45:56.230 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:56 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:45:56.606 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[80286]: logger=settings t=2024-04-26T04:45:56.229622222Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:45:56.606 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:56 smithi055 podman[80302]: 2024-04-26 04:45:56.278758744 +0000 UTC m=+0.032044206 container died 842bb596357f6c8ec0beb9edd3c245ccc4d04942d32f3210a4a0d6b8fb81fa0e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, version=8.5, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, maintainer=Paul Cuzner , release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container) 2024-04-26T04:45:56.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 18 16 43205 43189 149.949 99.9375 0.00496522 0.00666413 2024-04-26T04:45:57.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:56 smithi055 podman[80302]: 2024-04-26 04:45:56.886261192 +0000 UTC m=+0.639546665 container remove 842bb596357f6c8ec0beb9edd3c245ccc4d04942d32f3210a4a0d6b8fb81fa0e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., maintainer=Paul Cuzner , architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public) 2024-04-26T04:45:57.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:56 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:45:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:45:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:45:56] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-04-26T04:45:57.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:57 smithi055 ceph-mon[30328]: pgmap v34: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-26T04:45:57.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:57 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/305206998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:57.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:57 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:45:57.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:45:57 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.055s CPU time. 2024-04-26T04:45:57.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:57 smithi038 ceph-mon[26454]: pgmap v34: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-26T04:45:57.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:57 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/305206998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:57 smithi038 ceph-mon[36743]: pgmap v34: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-26T04:45:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:57 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/305206998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:45:57.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 19 16 45068 45052 148.184 116.438 0.00897498 0.00674312 2024-04-26T04:45:58.771 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[26454]: Upgrade: Updating mon.a 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[26454]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[26454]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[26454]: Deploying daemon mon.a on smithi038 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[36743]: Upgrade: Updating mon.a 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:45:58.772 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[36743]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:58.773 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:58 smithi038 ceph-mon[36743]: Deploying daemon mon.a on smithi038 2024-04-26T04:45:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-mon[30328]: Upgrade: Updating mon.a 2024-04-26T04:45:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='mgr.y' 2024-04-26T04:45:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:45:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:45:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-mon[30328]: from='mgr.25121 172.21.15.38:0/1792602913' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:45:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-mon[30328]: Deploying daemon mon.a on smithi038 2024-04-26T04:45:58.857 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:45:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:45:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:45:58.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:45:58.952155+0000 min lat: 0.0011979 max lat: 0.155299 avg lat: 0.00683337 2024-04-26T04:45:58.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:45:58.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 20 16 46808 46792 146.212 108.75 0.0223226 0.00683337 2024-04-26T04:45:59.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:45:59 smithi055 ceph-mon[30328]: pgmap v35: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-26T04:45:59.948 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:45:59 smithi038 ceph-mon[36743]: pgmap v35: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-26T04:45:59.948 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:59 smithi038 ceph-mon[26454]: pgmap v35: 89 pgs: 89 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2024-04-26T04:45:59.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 21 16 48396 48380 143.974 99.25 0.0134912 0.00694021 2024-04-26T04:46:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:45:59 smithi038 systemd[1]: Stopping Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:46:00.814 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:00 smithi038 ceph-mon[26454]: from='client.? 172.21.15.38:0/2298250091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:00.814 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:00 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/2298250091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:00 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/2298250091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:00.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 22 16 50096 50080 142.258 106.25 0.0135589 0.00702339 2024-04-26T04:46:01.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:46:00.810+0000 7fc82e403700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:46:01.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[26450]: 2024-04-26T04:46:00.810+0000 7fc82e403700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-26T04:46:01.130 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:00 smithi038 podman[93011]: 2024-04-26 04:46:00.92627591 +0000 UTC m=+0.669179851 container died 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c) 2024-04-26T04:46:01.409 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:01 smithi038 podman[93011]: 2024-04-26 04:46:01.408814602 +0000 UTC m=+1.151718547 container cleanup 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-26T04:46:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:01 smithi038 bash[93011]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a 2024-04-26T04:46:01.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 23 16 51719 51703 140.483 101.438 0.00260372 0.00710849 2024-04-26T04:46:02.431 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:02 smithi038 podman[93021]: 2024-04-26 04:46:02.153113196 +0000 UTC m=+1.219804070 container remove 0e262e788447e0e69cbd0be6fda891d4902ce28a8b4509e258be90f12b836f1a (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True) 2024-04-26T04:46:02.431 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:02 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-26T04:46:02.431 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:02 smithi038 systemd[1]: Stopped Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:02.431 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:02 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service: Consumed 16.882s CPU time. 2024-04-26T04:46:02.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 24 16 52990 52974 137.939 79.4375 0.0141548 0.00724429 2024-04-26T04:46:03.087 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:02 smithi038 systemd[1]: Starting Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:46:03.408 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 podman[93157]: 2024-04-26 04:46:02.992777723 +0000 UTC m=+0.047142923 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:03.409 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 podman[93157]: 2024-04-26 04:46:03.138260723 +0000 UTC m=+0.192625893 container create c46affca8e4e443fff694ab6eb4fb3ad0e534efdc57032a0f05252ae3ec149c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 podman[93157]: 2024-04-26 04:46:03.408220581 +0000 UTC m=+0.462585757 container init c46affca8e4e443fff694ab6eb4fb3ad0e534efdc57032a0f05252ae3ec149c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 podman[93157]: 2024-04-26 04:46:03.418690131 +0000 UTC m=+0.473055297 container start c46affca8e4e443fff694ab6eb4fb3ad0e534efdc57032a0f05252ae3ec149c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: pidfile_write: ignore empty --pid-file 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: load: jerasure load: lrc 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Git sha 0 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: DB SUMMARY 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: DB Session ID: GZAR73L18UFUCQEQ8M19 2024-04-26T04:46:03.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1157 Bytes 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2642854 ; 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.env: 0x5570b338fc60 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.info_log: 0x5570b53dfaa0 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.statistics: (nil) 2024-04-26T04:46:03.728 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.use_fsync: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.db_log_dir: 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.wal_dir: 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:46:03.729 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.write_buffer_manager: 0x5570b53b21e0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:46:03.730 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.unordered_write: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.row_cache: None 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.wal_filter: None 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.wal_compression: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:46:03.731 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:46:03.732 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_open_files: -1 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Compression algorithms supported: 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kZSTD supported: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:46:03.733 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:46:03.734 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:46:03.734 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:46:03.734 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.merge_operator: 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_filter: None 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5570b53dfba0) 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:46:03.735 INFO:journalctl@ceph.mon.a.smithi038.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: index_type: 0 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: data_block_index_type: 0 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: index_shortening: 1 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: checksum: 4 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: no_block_cache: 0 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache: 0x5570b53e3090 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache_options: 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: capacity : 536870912 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: num_shard_bits : 4 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: strict_capacity_limit : 0 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:46:03.736 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_cache_compressed: (nil) 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: persistent_cache: (nil) 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_size: 4096 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_size_deviation: 10 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_restart_interval: 16 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: index_block_restart_interval: 1 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: metadata_block_size: 4096 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: partition_filters: 0 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: use_delta_encoding: 1 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: filter_policy: bloomfilter 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: whole_key_filtering: 1 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: verify_compression: 0 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: format_version: 5 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: enable_index_compression: 1 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: block_align: 0 2024-04-26T04:46:03.737 INFO:journalctl@ceph.mon.a.smithi038.stdout: max_auto_readahead_size: 262144 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout: prepopulate_block_cache: 0 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression: NoCompression 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.num_levels: 7 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:46:03.738 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:46:03.739 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:46:03.740 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:46:03.741 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.ttl: 2592000 2024-04-26T04:46:03.742 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12062, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 553bf848-7d0a-42c2-9968-19ee9f0609ad 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106763476042, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106763490403, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2462521, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12056, "largest_seqno": 13196, "table_properties": {"data_size": 2456416, "index_size": 3742, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 12368, "raw_average_key_size": 23, "raw_value_size": 2444788, "raw_average_value_size": 4630, "num_data_blocks": 171, "num_entries": 528, "num_filter_entries": 528, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106763, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "553bf848-7d0a-42c2-9968-19ee9f0609ad", "db_session_id": "GZAR73L18UFUCQEQ8M19", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106763490753, "job": 1, "event": "recovery_finished"} 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5570b54d4000 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: DB pointer 0x5570b54be000 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** DB Stats ** 2024-04-26T04:46:03.744 INFO:journalctl@ceph.mon.a.smithi038.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: L0 1/0 2.35 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 305.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: L6 1/0 5.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Sum 2/0 8.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 305.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 305.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-26T04:46:03.745 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 305.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: Cumulative compaction: 0.00 GB write, 76.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:46:03.746 INFO:journalctl@ceph.mon.a.smithi038.stdout: Interval compaction: 0.00 GB write, 76.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: Block cache BinnedLRUCache@0x5570b53e3090#2 capacity: 512.00 MB usage: 58.91 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: Block cache entry stats(count,size,portion): DataBlock(4,28.31 KB,0.00540018%) FilterBlock(2,10.44 KB,0.0019908%) IndexBlock(2,20.16 KB,0.0038445%) Misc(1,0.00 KB,0%) 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: starting mon.a rank 0 at public addrs [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] at bind addrs [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???) e3 preinit fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).mds e9 new map 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).mds e9 print_map 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: e9 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: legacy client fscid: 1 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.747 INFO:journalctl@ceph.mon.a.smithi038.stdout: Filesystem 'foo' (1) 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: fs_name foo 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: epoch 8 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: created 2024-04-26T04:39:56.305501+0000 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: modified 2024-04-26T04:40:12.113527+0000 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: tableserver 0 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: root 0 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: session_timeout 60 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: session_autoclose 300 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: max_file_size 1099511627776 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: max_xattr_size 65536 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: required_client_features {} 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: last_failure 0 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: last_failure_osd_epoch 0 2024-04-26T04:46:03.748 INFO:journalctl@ceph.mon.a.smithi038.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: max_mds 1 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: in 0 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: up {0=24554} 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: failed 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: damaged 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: stopped 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: data_pools [4] 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: metadata_pool 3 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: inline_data disabled 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: balancer 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: bal_rank_mask -1 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: standby_count_wanted 1 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: [mds.foo.smithi038.qsffiu{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:46:03.749 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout: Standby daemons: 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout: 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout: [mds.foo.smithi055.cuaegn{-1:24532} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).osd e81 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).osd e81 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 ceph-mon[93172]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 bash[93157]: c46affca8e4e443fff694ab6eb4fb3ad0e534efdc57032a0f05252ae3ec149c8 2024-04-26T04:46:03.750 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:03 smithi038 systemd[1]: Started Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:03.952 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 25 16 54614 54598 136.482 101.5 0.00707116 0.00732225 2024-04-26T04:46:04.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ignoring --setuser ceph since I am not root 2024-04-26T04:46:04.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ignoring --setgroup ceph since I am not root 2024-04-26T04:46:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: pgmap v36: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: pgmap v37: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3777162107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: mon.a calling monitor election 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: monmap epoch 3 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: last_changed 2024-04-26T04:34:38.393611+0000 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: min_mon_release 17 (quincy) 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: election_strategy: 1 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:46:04.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: mgrmap e32: y(active, since 55s), standbys: x 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: daemon grafana.a on smithi055 is in unknown state 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: from='mgr.25121 ' entity='' 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[93172]: mgrmap e33: y(active, since 55s), standbys: x 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: pgmap v36: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: pgmap v37: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3777162107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: mon.a calling monitor election 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: monmap epoch 3 2024-04-26T04:46:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: last_changed 2024-04-26T04:34:38.393611+0000 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: min_mon_release 17 (quincy) 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: election_strategy: 1 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: mgrmap e32: y(active, since 55s), standbys: x 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: daemon grafana.a on smithi055 is in unknown state 2024-04-26T04:46:04.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: from='mgr.25121 ' entity='' 2024-04-26T04:46:04.478 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-mon[36743]: mgrmap e33: y(active, since 55s), standbys: x 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: ignoring --setuser ceph since I am not root 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: ignoring --setgroup ceph since I am not root 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:04.429+0000 7f6501b44200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: pgmap v36: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: pgmap v37: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3777162107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: mon.a calling monitor election 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:46:04.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: monmap epoch 3 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: last_changed 2024-04-26T04:34:38.393611+0000 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: min_mon_release 17 (quincy) 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: election_strategy: 1 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: osdmap e81: 8 total, 8 up, 8 in 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: mgrmap e32: y(active, since 55s), standbys: x 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: daemon grafana.a on smithi055 is in unknown state 2024-04-26T04:46:04.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: from='mgr.25121 ' entity='' 2024-04-26T04:46:04.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-mon[30328]: mgrmap e33: y(active, since 55s), standbys: x 2024-04-26T04:46:04.878 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:04.519+0000 7fa1f73e1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-26T04:46:04.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 26 16 55859 55843 134.225 77.8125 0.0146821 0.00744412 2024-04-26T04:46:05.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:04.681+0000 7f6501b44200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:46:05.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:04 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:04.827+0000 7f6501b44200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:46:05.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:04.876+0000 7fa1f73e1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:46:05.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:05.093+0000 7fa1f73e1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-26T04:46:05.266 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 1 2024-04-26T04:46:05.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:05 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:05.290+0000 7f6501b44200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:46:05.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:05 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:05.412+0000 7f6501b44200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:46:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:05 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/88932502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:05.648 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:05 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/88932502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:05.648 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:05 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/88932502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:46:05.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 27 16 57052 57036 132.015 74.5625 0.00439032 0.00756931 2024-04-26T04:46:05.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:05.703+0000 7fa1f73e1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-26T04:46:05.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:05.867+0000 7fa1f73e1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-26T04:46:06.133 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:46:06.712 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:06.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 28 16 58233 58217 129.936 73.8125 0.0229765 0.00768995 2024-04-26T04:46:07.498 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:07 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:07.204+0000 7f6501b44200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:46:07.499 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:07 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:07.497+0000 7f6501b44200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:46:07.499 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:07 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-26T04:46:07.499 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:07 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:07.499 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:07 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.055s CPU time. 2024-04-26T04:46:07.499 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:07 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:46:07.751 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:07 smithi055 podman[80427]: 2024-04-26 04:46:07.596172098 +0000 UTC m=+0.036194337 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:46:07.751 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:07 smithi055 podman[80427]: 2024-04-26 04:46:07.7506775 +0000 UTC m=+0.190699724 container create 1f9e593fa692fc52a7da71d2d9f095437b584ce06784c1dc1672f670271d62ac (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, build-date=2022-04-29T11:48:11.055897, vcs-type=git, io.openshift.tags=base rhel8, distribution-scope=public, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner ) 2024-04-26T04:46:07.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 29 16 59573 59557 128.343 83.75 0.00447395 0.00778442 2024-04-26T04:46:08.021 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:07 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:07.771+0000 7f6501b44200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:46:08.269 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:07 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:07.971+0000 7fa1f73e1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-26T04:46:08.286 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:08 smithi055 podman[80427]: 2024-04-26 04:46:08.021368418 +0000 UTC m=+0.461390655 container init 1f9e593fa692fc52a7da71d2d9f095437b584ce06784c1dc1672f670271d62ac (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, version=8.5, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.component=ubi8-container, release=239.1651231664, vcs-type=git, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-26T04:46:08.286 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:08 smithi055 podman[80427]: 2024-04-26 04:46:08.028363238 +0000 UTC m=+0.468385465 container start 1f9e593fa692fc52a7da71d2d9f095437b584ce06784c1dc1672f670271d62ac (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, distribution-scope=public, maintainer=Paul Cuzner ) 2024-04-26T04:46:08.286 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:08 smithi055 bash[80427]: 1f9e593fa692fc52a7da71d2d9f095437b584ce06784c1dc1672f670271d62ac 2024-04-26T04:46:08.286 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:08 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:08.558 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:08.267+0000 7fa1f73e1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-26T04:46:08.568 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:08.284+0000 7f6501b44200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:46:08.568 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:08.468+0000 7f6501b44200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:46:08.569 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[80442]: logger=settings t=2024-04-26T04:46:08.36621035Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:46:08.569 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:08 smithi055 podman[80463]: 2024-04-26 04:46:08.41869621 +0000 UTC m=+0.038584993 container died 1f9e593fa692fc52a7da71d2d9f095437b584ce06784c1dc1672f670271d62ac (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vendor=Red Hat, Inc., name=ubi8, io.openshift.expose-services=, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner ) 2024-04-26T04:46:08.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:08.807+0000 7f6501b44200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:46:08.871 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:08 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:08.556+0000 7fa1f73e1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-26T04:46:08.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 30 16 60809 60793 126.64 77.25 0.00694953 0.0078909 2024-04-26T04:46:09.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:09.069+0000 7fa1f73e1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-26T04:46:09.335 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:09 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:09.103+0000 7f6501b44200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:46:09.582 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:09.242+0000 7fa1f73e1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-26T04:46:09.583 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:09.580+0000 7fa1f73e1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-26T04:46:09.606 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:09 smithi055 podman[80463]: 2024-04-26 04:46:09.335320795 +0000 UTC m=+0.955209559 container remove 1f9e593fa692fc52a7da71d2d9f095437b584ce06784c1dc1672f670271d62ac (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, description=Ceph Grafana Container, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, distribution-scope=public, name=ubi8, version=8.5) 2024-04-26T04:46:09.607 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:09 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:46:09.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 31 16 62063 62047 125.083 78.375 0.0103534 0.00798967 2024-04-26T04:46:10.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:09 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:46:10.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:09 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.129s CPU time. 2024-04-26T04:46:10.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:09 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:09.933+0000 7fa1f73e1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-26T04:46:10.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:10 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:10.136+0000 7f6501b44200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:46:10.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 32 16 63459 63443 123.9 87.25 0.0068148 0.00806553 2024-04-26T04:46:11.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:10 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:10.826+0000 7f6501b44200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:46:11.106 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:10 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:10.979+0000 7f6501b44200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:46:11.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:10.995+0000 7fa1f73e1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-26T04:46:11.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:11 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:11.326+0000 7f6501b44200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:46:11.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 33 16 64842 64826 122.765 86.4375 0.0043581 0.00814017 2024-04-26T04:46:12.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:11 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:11.798+0000 7fa1f73e1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-26T04:46:12.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:11 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:11.974+0000 7fa1f73e1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-26T04:46:12.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:12.158+0000 7f6501b44200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:46:12.606 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:12.310+0000 7f6501b44200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:46:12.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:46:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:46:12.321Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T04:46:12.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:46:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:46:12.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T04:46:12.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:12.354+0000 7fa1f73e1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-26T04:46:12.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 34 16 66185 66169 121.622 83.9375 0.011254 0.0082168 2024-04-26T04:46:13.069 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:12.694+0000 7f6501b44200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:46:13.069 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:12 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:12.912+0000 7f6501b44200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:46:13.069 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:13.066+0000 7f6501b44200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:46:13.358 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:13.250+0000 7f6501b44200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:46:13.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:13.285+0000 7fa1f73e1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-26T04:46:13.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:13.456+0000 7fa1f73e1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-26T04:46:13.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:13 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:13.398+0000 7f6501b44200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:46:13.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 35 16 67527 67511 120.544 83.875 0.0131146 0.00829048 2024-04-26T04:46:14.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:13.914+0000 7fa1f73e1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-26T04:46:14.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:14.159+0000 7fa1f73e1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-26T04:46:14.720 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:14.335+0000 7fa1f73e1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-26T04:46:14.720 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:14.544+0000 7fa1f73e1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-26T04:46:14.721 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:14.719+0000 7fa1f73e1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-26T04:46:14.953 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 36 16 68784 68768 119.377 78.5625 0.0148708 0.00837151 2024-04-26T04:46:15.056 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:14 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:14.690+0000 7f6501b44200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:46:15.357 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:15.054+0000 7f6501b44200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: [26/Apr/2024:04:46:15] ENGINE Bus STARTING 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: CherryPy Checker: 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: The Application mounted at '' has an empty config. 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: [26/Apr/2024:04:46:15] ENGINE Serving on http://:::9283 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x[76772]: [26/Apr/2024:04:46:15] ENGINE Bus STARTED 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-mon[30328]: Standby manager daemon x restarted 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-mon[30328]: Standby manager daemon x started 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:46:15.408 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:46:15.409 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:46:15.409 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:15 smithi055 ceph-mon[30328]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:46:15.447 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[93172]: Standby manager daemon x restarted 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[93172]: Standby manager daemon x started 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[93172]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[93172]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[93172]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[93172]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[36743]: Standby manager daemon x restarted 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[36743]: Standby manager daemon x started 2024-04-26T04:46:15.448 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-26T04:46:15.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-26T04:46:15.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-26T04:46:15.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:15 smithi038 ceph-mon[36743]: from='mgr.? 172.21.15.55:0/634259408' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-26T04:46:15.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 37 16 70076 70060 118.333 80.75 0.0164632 0.00844532 2024-04-26T04:46:16.456 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-mon[93172]: mgrmap e34: y(active, since 66s), standbys: x 2024-04-26T04:46:16.456 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:16.173+0000 7fa1f73e1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-26T04:46:16.456 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-mon[36743]: mgrmap e34: y(active, since 66s), standbys: x 2024-04-26T04:46:16.457 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:46:16.350Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=5 2024-04-26T04:46:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:16 smithi055 ceph-mon[30328]: mgrmap e34: y(active, since 66s), standbys: x 2024-04-26T04:46:16.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:16.586+0000 7fa1f73e1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-26T04:46:16.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 38 16 71286 71270 117.209 75.625 0.0188992 0.0085258 2024-04-26T04:46:17.043 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:46:16] ENGINE Bus STARTING 2024-04-26T04:46:17.043 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:46:16] "GET /metrics HTTP/1.1" 200 33400 "" "Prometheus/2.43.0" 2024-04-26T04:46:17.043 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: CherryPy Checker: 2024-04-26T04:46:17.044 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: The Application mounted at '' has an empty config. 2024-04-26T04:46:17.044 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.044 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:46:17] ENGINE Serving on http://:::9283 2024-04-26T04:46:17.240 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: Active manager daemon y restarted 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: Activating manager daemon y 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: mgrmap e35: y(active, starting, since 0.0136287s), standbys: x 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:46:17.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: Manager daemon y is now available 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:46:17.242 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:46:17.299 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:04:46:17] ENGINE Bus STARTED 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.084+0000 7fa159e96700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.084+0000 7fa159e96700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.084+0000 7fa159e96700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.084+0000 7fa159e96700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.084+0000 7fa159e96700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.104+0000 7fa153d4a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.104+0000 7fa153d4a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.104+0000 7fa153d4a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.104+0000 7fa153d4a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.300 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.104+0000 7fa153d4a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: Active manager daemon y restarted 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: Activating manager daemon y 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: mgrmap e35: y(active, starting, since 0.0136287s), standbys: x 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:46:17.301 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: Manager daemon y is now available 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:46:17.302 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: Active manager daemon y restarted 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: Activating manager daemon y 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: mgrmap e35: y(active, starting, since 0.0136287s), standbys: x 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.cuaegn"}]: dispatch 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi038.qsffiu"}]: dispatch 2024-04-26T04:46:17.303 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: Manager daemon y is now available 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:17.304 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:17.305 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-26T04:46:17.305 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-26T04:46:17.665 INFO:teuthology.orchestra.run.smithi038.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:46:17.665 INFO:teuthology.orchestra.run.smithi038.stdout:alertmanager.a smithi038 *:9093,9094 running (2m) 62s ago 7m 19.3M - 0.25.0 c8568f914cd2 bd09134fac6b 2024-04-26T04:46:17.665 INFO:teuthology.orchestra.run.smithi038.stdout:grafana.a smithi055 *:3000 unknown 32s ago 6m - - 954c08fa6188 2d3a8e32d944 2024-04-26T04:46:17.665 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi038.qsffiu smithi038 running (6m) 62s ago 6m 15.7M - 17.2.7-902-gc61472c8 0de407c1b3af 841ef3f1e866 2024-04-26T04:46:17.665 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi055.cuaegn smithi055 running (6m) 32s ago 6m 12.9M - 17.2.7-902-gc61472c8 0de407c1b3af 030d9ab50e92 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.x smithi055 *:8443,9283,8765 running (40s) 32s ago 11m 67.8M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f26c41ca1326 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.y smithi038 *:8443,9283,8765 running (84s) 62s ago 13m 476M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4c95c5f5ae02 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:mon.a smithi038 running (13m) 62s ago 13m 49.3M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 0e262e788447 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:mon.b smithi055 running (11m) 32s ago 11m 46.7M 2048M 17.2.7-902-gc61472c8 0de407c1b3af e2ab87d945f4 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:mon.c smithi038 running (11m) 62s ago 11m 41.4M 2048M 17.2.7-902-gc61472c8 0de407c1b3af 96c89d482073 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.a smithi038 *:9100 running (2m) 62s ago 7m 8921k - 1.5.0 0da6a335fe13 f3d04e42cf8f 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.b smithi055 *:9100 running (110s) 32s ago 7m 9042k - 1.5.0 0da6a335fe13 9087143d9886 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:osd.0 smithi038 running (11m) 62s ago 11m 384M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 7a0711df145c 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:osd.1 smithi038 running (10m) 62s ago 10m 252M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 54e6c9519bdd 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:osd.2 smithi038 running (10m) 62s ago 10m 323M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 05094c28a4cd 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:osd.3 smithi038 running (9m) 62s ago 9m 169M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 61ce04224439 2024-04-26T04:46:17.666 INFO:teuthology.orchestra.run.smithi038.stdout:osd.4 smithi055 running (9m) 32s ago 9m 291M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 40ece9b7ce4f 2024-04-26T04:46:17.667 INFO:teuthology.orchestra.run.smithi038.stdout:osd.5 smithi055 running (9m) 32s ago 9m 321M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 50fe68fbb905 2024-04-26T04:46:17.667 INFO:teuthology.orchestra.run.smithi038.stdout:osd.6 smithi055 running (8m) 32s ago 8m 173M 2481M 17.2.7-902-gc61472c8 0de407c1b3af bc76cfe8fbcd 2024-04-26T04:46:17.667 INFO:teuthology.orchestra.run.smithi038.stdout:osd.7 smithi055 running (8m) 32s ago 8m 207M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 6ca051d18a67 2024-04-26T04:46:17.667 INFO:teuthology.orchestra.run.smithi038.stdout:prometheus.a smithi055 *:9095 running (46s) 32s ago 7m 42.5M - 2.43.0 a07b618ecd1d c40205ac5452 2024-04-26T04:46:17.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 39 16 72229 72213 115.715 58.9375 0.00855571 0.00863442 2024-04-26T04:46:17.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:17 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:46:17.633+0000 7fa1d4887700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-26T04:46:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-mon[93172]: mgrmap e36: y(active, since 1.02115s), standbys: x 2024-04-26T04:46:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-mon[93172]: from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-mon[93172]: pgmap v3: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:18.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-mon[36743]: mgrmap e36: y(active, since 1.02115s), standbys: x 2024-04-26T04:46:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-mon[36743]: from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-mon[36743]: pgmap v3: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:18.725 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 04:46:18 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T04:46:18.514Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2024-04-26T04:46:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:18 smithi055 ceph-mon[30328]: mgrmap e36: y(active, since 1.02115s), standbys: x 2024-04-26T04:46:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:18 smithi055 ceph-mon[30328]: from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:18 smithi055 ceph-mon[30328]: pgmap v3: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:18.856 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:46:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:46:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:18.878 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:46:18.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:46:18.954358+0000 min lat: 0.0011979 max lat: 0.155299 avg lat: 0.00871608 2024-04-26T04:46:18.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:46:18.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 40 16 73406 73390 114.66 73.5625 0.0139292 0.00871608 2024-04-26T04:46:19.871 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:19 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-26T04:46:19.871 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:19 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:19.871 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:19 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.129s CPU time. 2024-04-26T04:46:19.871 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:19 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:46:19.872 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: [26/Apr/2024:04:46:18] ENGINE Bus STARTING 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: Cluster is now healthy 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: pgmap v4: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: [26/Apr/2024:04:46:18] ENGINE Serving on http://172.21.15.38:8765 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: [26/Apr/2024:04:46:18] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:46:19.873 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:19 smithi055 ceph-mon[30328]: [26/Apr/2024:04:46:18] ENGINE Bus STARTED 2024-04-26T04:46:19.945 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: [26/Apr/2024:04:46:18] ENGINE Bus STARTING 2024-04-26T04:46:19.945 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:46:19.945 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: Cluster is now healthy 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: pgmap v4: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: [26/Apr/2024:04:46:18] ENGINE Serving on http://172.21.15.38:8765 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: [26/Apr/2024:04:46:18] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[93172]: [26/Apr/2024:04:46:18] ENGINE Bus STARTED 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: [26/Apr/2024:04:46:18] ENGINE Bus STARTING 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: Cluster is now healthy 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: mgrmap e37: y(active, since 2s), standbys: x 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: pgmap v4: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: [26/Apr/2024:04:46:18] ENGINE Serving on http://172.21.15.38:8765 2024-04-26T04:46:19.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: [26/Apr/2024:04:46:18] ENGINE Serving on https://172.21.15.38:7150 2024-04-26T04:46:19.947 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:19 smithi038 ceph-mon[36743]: [26/Apr/2024:04:46:18] ENGINE Bus STARTED 2024-04-26T04:46:19.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 41 16 74499 74483 113.53 68.3125 0.00480299 0.00880317 2024-04-26T04:46:20.183 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:20.474 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 podman[80917]: 2024-04-26 04:46:20.0854293 +0000 UTC m=+0.027260222 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:46:20.474 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 podman[80917]: 2024-04-26 04:46:20.20391817 +0000 UTC m=+0.145749070 container create 363590f247dfff533383614feb4bffe7be5990dbacac332029229f3dd95fb9de (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , name=ubi8, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:46:20.736 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 podman[80917]: 2024-04-26 04:46:20.473559342 +0000 UTC m=+0.415390253 container init 363590f247dfff533383614feb4bffe7be5990dbacac332029229f3dd95fb9de (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, maintainer=Paul Cuzner , io.buildah.version=1.32.0, description=Ceph Grafana Container, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=239.1651231664, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-26T04:46:20.736 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 podman[80917]: 2024-04-26 04:46:20.47751443 +0000 UTC m=+0.419345331 container start 363590f247dfff533383614feb4bffe7be5990dbacac332029229f3dd95fb9de (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.buildah.version=1.32.0, description=Ceph Grafana Container) 2024-04-26T04:46:20.736 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 bash[80917]: 363590f247dfff533383614feb4bffe7be5990dbacac332029229f3dd95fb9de 2024-04-26T04:46:20.736 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:20.736 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:20 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[80945]: logger=settings t=2024-04-26T04:46:20.735685439Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:46:20.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 42 15 75564 75549 112.413 66.625 0.0176764 0.00889063 2024-04-26T04:46:21.275 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:21 smithi055 podman[80983]: 2024-04-26 04:46:21.009641693 +0000 UTC m=+0.264219575 container died 363590f247dfff533383614feb4bffe7be5990dbacac332029229f3dd95fb9de (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, name=ubi8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, description=Ceph Grafana Container, io.buildah.version=1.32.0) 2024-04-26T04:46:21.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 43 16 76821 76805 111.624 78.5 0.0039489 0.00895172 2024-04-26T04:46:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:21 smithi038 ceph-mon[93172]: pgmap v5: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:21 smithi038 ceph-mon[93172]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:46:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:21 smithi038 ceph-mon[36743]: pgmap v5: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:21 smithi038 ceph-mon[36743]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:46:22.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:21 smithi055 podman[80983]: 2024-04-26 04:46:21.614433586 +0000 UTC m=+0.869011445 container remove 363590f247dfff533383614feb4bffe7be5990dbacac332029229f3dd95fb9de (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, vcs-type=git, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, name=ubi8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, distribution-scope=public) 2024-04-26T04:46:22.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:21 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:46:22.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:21 smithi055 ceph-mon[30328]: pgmap v5: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail 2024-04-26T04:46:22.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:21 smithi055 ceph-mon[30328]: mgrmap e38: y(active, since 4s), standbys: x 2024-04-26T04:46:22.655 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:22 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:46:22.655 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:22 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.046s CPU time. 2024-04-26T04:46:22.954 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 44 16 78073 78057 110.865 78.25 0.00774913 0.00901472 2024-04-26T04:46:23.926 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:23 smithi038 ceph-mon[36743]: pgmap v6: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 KiB/s rd, 35 MiB/s wr, 587 op/s 2024-04-26T04:46:23.926 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:23 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:23.926 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:23 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:23.927 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:23 smithi038 ceph-mon[93172]: pgmap v6: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 KiB/s rd, 35 MiB/s wr, 587 op/s 2024-04-26T04:46:23.927 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:23.927 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:23.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 45 16 79290 79274 110.092 76.0625 0.0121283 0.00907834 2024-04-26T04:46:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:23 smithi055 ceph-mon[30328]: pgmap v6: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41 KiB/s rd, 35 MiB/s wr, 587 op/s 2024-04-26T04:46:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:23 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:23 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:24.588 INFO:teuthology.orchestra.run.smithi038.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:24.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 46 16 80439 80423 109.26 71.8125 0.00911335 0.00914785 2024-04-26T04:46:25.433 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:46:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[93172]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[93172]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[93172]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 51 MiB/s wr, 835 op/s 2024-04-26T04:46:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[36743]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:25.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[36743]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:25.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[36743]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 51 MiB/s wr, 835 op/s 2024-04-26T04:46:25.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:25 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:25 smithi055 ceph-mon[30328]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:46:25.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:25 smithi055 ceph-mon[30328]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:46:25.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:25 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:25 smithi055 ceph-mon[30328]: pgmap v7: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 30 KiB/s rd, 51 MiB/s wr, 835 op/s 2024-04-26T04:46:25.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:25 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:25 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:25.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 47 16 81611 81595 108.493 73.25 0.0121451 0.00921185 2024-04-26T04:46:26.504 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:46:26.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 48 16 82833 82817 107.824 76.375 0.00468934 0.00926847 2024-04-26T04:46:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:46:26] "GET /metrics HTTP/1.1" 200 33400 "" "Prometheus/2.43.0" 2024-04-26T04:46:27.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 49 16 84084 84068 107.219 78.1875 0.00576072 0.0093218 2024-04-26T04:46:27.970 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:27 smithi055 ceph-mon[30328]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 40 MiB/s wr, 649 op/s 2024-04-26T04:46:27.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:27 smithi038 ceph-mon[93172]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 40 MiB/s wr, 649 op/s 2024-04-26T04:46:27.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:27 smithi038 ceph-mon[36743]: pgmap v8: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 23 KiB/s rd, 40 MiB/s wr, 649 op/s 2024-04-26T04:46:28.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 50 15 85204 85189 106.476 70.0625 0.0197655 0.00938652 2024-04-26T04:46:29.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:46:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:46:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:29 smithi055 ceph-mon[30328]: pgmap v9: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 19 KiB/s rd, 67 MiB/s wr, 1.08k op/s 2024-04-26T04:46:29.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 51 16 86307 86291 105.738 68.875 0.00746472 0.00945217 2024-04-26T04:46:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:29 smithi038 ceph-mon[93172]: pgmap v9: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 19 KiB/s rd, 67 MiB/s wr, 1.08k op/s 2024-04-26T04:46:29.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:29 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:29.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:29 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:29.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:29 smithi038 ceph-mon[36743]: pgmap v9: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 19 KiB/s rd, 67 MiB/s wr, 1.08k op/s 2024-04-26T04:46:30.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 52 16 87402 87386 105.021 68.4375 0.00818371 0.0095162 2024-04-26T04:46:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:31.367 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.367 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.367 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.367 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.367 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:31.367 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:31.368 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.368 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.368 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.368 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:31.368 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:31.368 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:31.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 53 16 88449 88433 104.274 65.4375 0.0100801 0.00958542 2024-04-26T04:46:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:46:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:46:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: pgmap v10: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 61 MiB/s wr, 987 op/s 2024-04-26T04:46:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:46:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:46:32.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:32.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: pgmap v10: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 61 MiB/s wr, 987 op/s 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.conf 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.conf 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: pgmap v10: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 61 MiB/s wr, 987 op/s 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:32.477 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:32 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:32.914 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:32 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-04-26T04:46:32.914 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:32 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:32.914 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:32 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.046s CPU time. 2024-04-26T04:46:32.914 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:32 smithi055 systemd[1]: Starting Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:46:32.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 54 16 89545 89529 103.611 68.5 0.00725609 0.00964605 2024-04-26T04:46:33.180 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:32 smithi055 podman[85089]: 2024-04-26 04:46:32.817279009 +0000 UTC m=+0.034442680 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-26T04:46:33.180 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:33 smithi055 podman[85089]: 2024-04-26 04:46:33.086170431 +0000 UTC m=+0.303334092 container create 58edd1739d1ad490c947605bfa95c00c503ad483be4896e78f76b4f7a8115fd4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, name=ubi8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, version=8.5, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, release=239.1651231664) 2024-04-26T04:46:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:46:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:46:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:46:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:46:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:46:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:33.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: Updating smithi038:/etc/ceph/ceph.client.admin.keyring 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: Updating smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: Updating smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:46:33.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:33 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:33.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 55 16 90753 90737 103.1 75.5 0.0100428 0.00969446 2024-04-26T04:46:34.434 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:34 smithi055 ceph-mon[30328]: pgmap v11: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 62 MiB/s wr, 994 op/s 2024-04-26T04:46:34.434 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:34 smithi055 ceph-mon[30328]: pgmap v12: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-04-26T04:46:34.434 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:34 smithi055 ceph-mon[30328]: pgmap v13: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-04-26T04:46:34.434 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:34 smithi055 ceph-mon[30328]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:46:34.434 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:34 smithi055 podman[85089]: 2024-04-26 04:46:34.16455268 +0000 UTC m=+1.381716342 container init 58edd1739d1ad490c947605bfa95c00c503ad483be4896e78f76b4f7a8115fd4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, io.openshift.tags=base rhel8, distribution-scope=public, io.buildah.version=1.32.0, vendor=Red Hat, Inc., maintainer=Paul Cuzner , description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-26T04:46:34.434 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:34 smithi055 podman[85089]: 2024-04-26 04:46:34.168636241 +0000 UTC m=+1.385799915 container start 58edd1739d1ad490c947605bfa95c00c503ad483be4896e78f76b4f7a8115fd4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.component=ubi8-container, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0) 2024-04-26T04:46:34.434 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:34 smithi055 bash[85089]: 58edd1739d1ad490c947605bfa95c00c503ad483be4896e78f76b4f7a8115fd4 2024-04-26T04:46:34.435 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:34 smithi055 systemd[1]: Started Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[36743]: pgmap v11: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 62 MiB/s wr, 994 op/s 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[36743]: pgmap v12: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[36743]: pgmap v13: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[36743]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[93172]: pgmap v11: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 62 MiB/s wr, 994 op/s 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[93172]: pgmap v12: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[93172]: pgmap v13: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-04-26T04:46:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:34 smithi038 ceph-mon[93172]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-26T04:46:34.796 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a[85134]: logger=settings t=2024-04-26T04:46:34.433419934Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-26T04:46:34.796 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:34 smithi055 podman[85151]: 2024-04-26 04:46:34.49089106 +0000 UTC m=+0.044005287 container died 58edd1739d1ad490c947605bfa95c00c503ad483be4896e78f76b4f7a8115fd4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.openshift.tags=base rhel8, release=239.1651231664, architecture=x86_64, vcs-type=git, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, name=ubi8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container) 2024-04-26T04:46:34.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 56 16 91964 91948 102.611 75.6875 0.00717913 0.00974091 2024-04-26T04:46:35.356 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:35 smithi055 podman[85151]: 2024-04-26 04:46:35.103383046 +0000 UTC m=+0.656497276 container remove 58edd1739d1ad490c947605bfa95c00c503ad483be4896e78f76b4f7a8115fd4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-grafana-a, vendor=Red Hat, Inc., release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.buildah.version=1.32.0, vcs-type=git, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.expose-services=, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64) 2024-04-26T04:46:35.356 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:35 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-26T04:46:35.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:35 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:46:35.856 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:35 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.048s CPU time. 2024-04-26T04:46:35.955 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 57 16 93150 93134 102.111 74.125 0.0165969 0.00978782 2024-04-26T04:46:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:36 smithi038 ceph-mon[36743]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-04-26T04:46:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:36 smithi038 ceph-mon[93172]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-04-26T04:46:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:36 smithi055 ceph-mon[30328]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-04-26T04:46:36.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 58 16 94340 94324 101.632 74.375 0.0149094 0.00983448 2024-04-26T04:46:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:46:36] "GET /metrics HTTP/1.1" 200 36119 "" "Prometheus/2.43.0" 2024-04-26T04:46:37.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 59 16 95508 95492 101.147 73 0.0206079 0.0098816 2024-04-26T04:46:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:38 smithi038 ceph-mon[93172]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s 2024-04-26T04:46:38.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:38 smithi038 ceph-mon[36743]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s 2024-04-26T04:46:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:38 smithi055 ceph-mon[30328]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s 2024-04-26T04:46:38.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:46:38.956121+0000 min lat: 0.0011979 max lat: 0.155299 avg lat: 0.00992834 2024-04-26T04:46:38.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:46:38.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 60 16 96668 96652 100.67 72.5 0.0166533 0.00992834 2024-04-26T04:46:39.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:46:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:46:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:39.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 61 16 97827 97811 100.207 72.4375 0.0172371 0.00997403 2024-04-26T04:46:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:40 smithi038 ceph-mon[93172]: pgmap v16: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-04-26T04:46:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:40 smithi038 ceph-mon[36743]: pgmap v16: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-04-26T04:46:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:40 smithi055 ceph-mon[30328]: pgmap v16: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-04-26T04:46:40.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 62 16 98948 98932 99.7203 70.0625 0.037257 0.010023 2024-04-26T04:46:41.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 63 16 100092 100076 99.2722 71.5 0.0232232 0.0100686 2024-04-26T04:46:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:42 smithi038 ceph-mon[93172]: pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-04-26T04:46:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:42 smithi038 ceph-mon[36743]: pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-04-26T04:46:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:42 smithi055 ceph-mon[30328]: pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-04-26T04:46:42.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 64 16 101198 101182 98.8011 69.125 0.0118588 0.0101157 2024-04-26T04:46:43.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 65 16 102344 102328 98.3829 71.625 0.0119151 0.0101593 2024-04-26T04:46:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:44 smithi038 ceph-mon[93172]: pgmap v18: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:46:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:44 smithi038 ceph-mon[36743]: pgmap v18: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:46:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:44 smithi055 ceph-mon[30328]: pgmap v18: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:46:44.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 66 16 103438 103422 97.9281 68.375 0.0158573 0.0102061 2024-04-26T04:46:45.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 67 16 104530 104514 97.4851 68.25 0.0192417 0.0102527 2024-04-26T04:46:46.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:45 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-04-26T04:46:46.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:45 smithi055 systemd[1]: Stopped Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:46.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:45 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Consumed 1.048s CPU time. 2024-04-26T04:46:46.106 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:45 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Start request repeated too quickly. 2024-04-26T04:46:46.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:45 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-26T04:46:46.107 INFO:journalctl@ceph.grafana.a.smithi055.stdout:Apr 26 04:46:45 smithi055 systemd[1]: Failed to start Ceph grafana.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:46:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:46 smithi055 ceph-mon[30328]: pgmap v19: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-04-26T04:46:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:46 smithi038 ceph-mon[93172]: pgmap v19: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-04-26T04:46:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:46 smithi038 ceph-mon[36743]: pgmap v19: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-04-26T04:46:46.956 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 68 16 105616 105600 97.0496 67.875 0.0188695 0.0102985 2024-04-26T04:46:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:46:46] "GET /metrics HTTP/1.1" 200 36120 "" "Prometheus/2.43.0" 2024-04-26T04:46:47.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 69 16 106691 106675 96.6167 67.1875 0.0188944 0.0103452 2024-04-26T04:46:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:47 smithi055 ceph-mon[30328]: pgmap v20: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-04-26T04:46:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:47 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:47 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:47 smithi038 ceph-mon[93172]: pgmap v20: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-04-26T04:46:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:47 smithi038 ceph-mon[36743]: pgmap v20: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-04-26T04:46:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:47 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:46:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:47 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:46:48.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 70 16 107777 107761 96.206 67.875 0.0303368 0.0103888 2024-04-26T04:46:49.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:46:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:46:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:49.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 71 16 108845 108829 95.7911 66.75 0.0243406 0.0104334 2024-04-26T04:46:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:49 smithi055 ceph-mon[30328]: pgmap v21: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-04-26T04:46:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:49 smithi038 ceph-mon[36743]: pgmap v21: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-04-26T04:46:50.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:49 smithi038 ceph-mon[93172]: pgmap v21: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-04-26T04:46:50.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 72 16 109909 109893 95.3842 66.5 0.0141818 0.0104787 2024-04-26T04:46:51.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 73 16 110836 110820 94.8711 57.9375 0.0128532 0.0105355 2024-04-26T04:46:52.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:51 smithi055 ceph-mon[30328]: pgmap v22: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-04-26T04:46:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:51 smithi038 ceph-mon[36743]: pgmap v22: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-04-26T04:46:52.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:51 smithi038 ceph-mon[93172]: pgmap v22: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2024-04-26T04:46:52.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 74 16 111734 111718 94.3475 56.125 0.0168768 0.010594 2024-04-26T04:46:53.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 75 16 112748 112732 93.9344 63.375 0.0219252 0.0106408 2024-04-26T04:46:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:53 smithi055 ceph-mon[30328]: pgmap v23: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-04-26T04:46:54.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:53 smithi038 ceph-mon[36743]: pgmap v23: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-04-26T04:46:54.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:53 smithi038 ceph-mon[93172]: pgmap v23: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-04-26T04:46:54.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 76 16 113759 113743 93.5298 63.1875 0.0273122 0.0106869 2024-04-26T04:46:55.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 77 16 114776 114760 93.1405 63.5625 0.0231792 0.010731 2024-04-26T04:46:56.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:55 smithi055 ceph-mon[30328]: pgmap v24: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:46:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:55 smithi038 ceph-mon[93172]: pgmap v24: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:46:56.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:55 smithi038 ceph-mon[36743]: pgmap v24: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:46:56.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 78 16 115791 115775 92.7596 63.4375 0.0260652 0.0107759 2024-04-26T04:46:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:46:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:46:56] "GET /metrics HTTP/1.1" 200 36120 "" "Prometheus/2.43.0" 2024-04-26T04:46:57.957 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 79 16 116792 116776 92.3773 62.5625 0.0146129 0.0108199 2024-04-26T04:46:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:57 smithi055 ceph-mon[30328]: pgmap v25: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:46:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:57 smithi038 ceph-mon[93172]: pgmap v25: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:46:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:57 smithi038 ceph-mon[36743]: pgmap v25: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:46:58.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:46:58.957953+0000 min lat: 0.0011979 max lat: 0.155299 avg lat: 0.0108641 2024-04-26T04:46:58.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:46:58.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 80 16 117787 117771 91.9999 62.1875 0.00843517 0.0108641 2024-04-26T04:46:59.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:46:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:46:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:46:59.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 81 16 118782 118766 91.6317 62.1875 0.0322941 0.0109081 2024-04-26T04:47:00.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:46:59 smithi055 ceph-mon[30328]: pgmap v26: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-26T04:47:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:46:59 smithi038 ceph-mon[93172]: pgmap v26: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-26T04:47:00.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:46:59 smithi038 ceph-mon[36743]: pgmap v26: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-26T04:47:00.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 82 16 119777 119761 91.2726 62.1875 0.0159104 0.0109506 2024-04-26T04:47:01.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 83 16 120767 120751 90.9184 61.875 0.011316 0.0109942 2024-04-26T04:47:02.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:01 smithi055 ceph-mon[30328]: pgmap v27: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2024-04-26T04:47:02.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:01 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:02.214 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:01 smithi038 ceph-mon[93172]: pgmap v27: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2024-04-26T04:47:02.215 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:02.215 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:01 smithi038 ceph-mon[36743]: pgmap v27: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2024-04-26T04:47:02.215 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:01 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:02.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 84 16 121748 121732 90.5658 61.3125 0.00911121 0.011036 2024-04-26T04:47:03.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 85 16 122703 122687 90.2025 59.6875 0.0168251 0.0110815 2024-04-26T04:47:04.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:03 smithi055 ceph-mon[30328]: pgmap v28: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:47:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:03 smithi038 ceph-mon[36743]: pgmap v28: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:47:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:03 smithi038 ceph-mon[93172]: pgmap v28: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-26T04:47:04.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 86 16 123676 123660 89.8607 60.8125 0.0115047 0.0111229 2024-04-26T04:47:05.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 87 16 124645 124629 89.5239 60.5625 0.0105556 0.0111646 2024-04-26T04:47:06.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:05 smithi038 ceph-mon[36743]: pgmap v29: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2024-04-26T04:47:06.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:05 smithi038 ceph-mon[93172]: pgmap v29: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2024-04-26T04:47:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:05 smithi055 ceph-mon[30328]: pgmap v29: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 57 MiB/s wr, 907 op/s 2024-04-26T04:47:06.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 88 16 125592 125576 89.1791 59.1875 0.0174402 0.011208 2024-04-26T04:47:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:47:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:47:06] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-04-26T04:47:07.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 89 16 126531 126515 88.8364 58.6875 0.0300813 0.0112514 2024-04-26T04:47:08.158 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:07 smithi055 ceph-mon[30328]: pgmap v30: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2024-04-26T04:47:08.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:07 smithi038 ceph-mon[36743]: pgmap v30: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2024-04-26T04:47:08.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:07 smithi038 ceph-mon[93172]: pgmap v30: 89 pgs: 89 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2024-04-26T04:47:08.958 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total time run: 90.0074 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total writes made: 127451 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Write size: 65536 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object size: 65536 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Bandwidth (MB/sec): 88.5003 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Bandwidth: 36.6615 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max bandwidth (MB/sec): 220.438 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min bandwidth (MB/sec): 56.125 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average IOPS: 1416 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev IOPS: 586.584 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max IOPS: 3527 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min IOPS: 898 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average Latency(s): 0.011295 2024-04-26T04:47:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Latency(s): 0.00712614 2024-04-26T04:47:08.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max latency(s): 0.155299 2024-04-26T04:47:08.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min latency(s): 0.0011979 2024-04-26T04:47:08.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Cleaning up (deleting benchmark objects) 2024-04-26T04:47:09.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:47:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:47:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:10.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:09 smithi038 ceph-mon[93172]: pgmap v31: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:47:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:09 smithi038 ceph-mon[36743]: pgmap v31: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:47:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:09 smithi055 ceph-mon[30328]: pgmap v31: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:47:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:11 smithi038 ceph-mon[93172]: pgmap v32: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2024-04-26T04:47:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:11 smithi038 ceph-mon[36743]: pgmap v32: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2024-04-26T04:47:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:11 smithi055 ceph-mon[30328]: pgmap v32: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s 2024-04-26T04:47:14.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:13 smithi038 ceph-mon[93172]: pgmap v33: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 56 MiB/s wr, 1.14k op/s 2024-04-26T04:47:14.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:13 smithi038 ceph-mon[36743]: pgmap v33: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 56 MiB/s wr, 1.14k op/s 2024-04-26T04:47:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:13 smithi055 ceph-mon[30328]: pgmap v33: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 56 MiB/s wr, 1.14k op/s 2024-04-26T04:47:16.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:15 smithi038 ceph-mon[93172]: pgmap v34: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 882 op/s 2024-04-26T04:47:16.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:15 smithi038 ceph-mon[36743]: pgmap v34: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 882 op/s 2024-04-26T04:47:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:15 smithi055 ceph-mon[30328]: pgmap v34: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 882 op/s 2024-04-26T04:47:17.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:16 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:16 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:47:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:47:16] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-04-26T04:47:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:16 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:18.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:17 smithi038 ceph-mon[93172]: pgmap v35: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 799 op/s 2024-04-26T04:47:18.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:17 smithi038 ceph-mon[36743]: pgmap v35: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 799 op/s 2024-04-26T04:47:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:17 smithi055 ceph-mon[30328]: pgmap v35: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 799 op/s 2024-04-26T04:47:19.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:47:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:47:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:20.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:19 smithi038 ceph-mon[93172]: pgmap v36: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 1.20k op/s 2024-04-26T04:47:20.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:19 smithi038 ceph-mon[36743]: pgmap v36: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 1.20k op/s 2024-04-26T04:47:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:19 smithi055 ceph-mon[30328]: pgmap v36: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 31 MiB/s wr, 1.20k op/s 2024-04-26T04:47:22.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:21 smithi038 ceph-mon[93172]: pgmap v37: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 6.2 MiB/s wr, 797 op/s 2024-04-26T04:47:22.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:21 smithi038 ceph-mon[36743]: pgmap v37: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 6.2 MiB/s wr, 797 op/s 2024-04-26T04:47:22.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:21 smithi055 ceph-mon[30328]: pgmap v37: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 6.2 MiB/s wr, 797 op/s 2024-04-26T04:47:24.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:23 smithi038 ceph-mon[93172]: pgmap v38: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 6.2 MiB/s wr, 1.02k op/s 2024-04-26T04:47:24.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:23 smithi038 ceph-mon[36743]: pgmap v38: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 6.2 MiB/s wr, 1.02k op/s 2024-04-26T04:47:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:23 smithi055 ceph-mon[30328]: pgmap v38: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 6.2 MiB/s wr, 1.02k op/s 2024-04-26T04:47:26.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:25 smithi038 ceph-mon[93172]: pgmap v39: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.1 MiB/s wr, 868 op/s 2024-04-26T04:47:26.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:25 smithi038 ceph-mon[36743]: pgmap v39: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.1 MiB/s wr, 868 op/s 2024-04-26T04:47:26.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:25 smithi055 ceph-mon[30328]: pgmap v39: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 1.1 MiB/s wr, 868 op/s 2024-04-26T04:47:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:47:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:47:26] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-04-26T04:47:28.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:27 smithi038 ceph-mon[93172]: pgmap v40: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-04-26T04:47:28.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:27 smithi038 ceph-mon[36743]: pgmap v40: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-04-26T04:47:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:27 smithi055 ceph-mon[30328]: pgmap v40: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2024-04-26T04:47:29.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:47:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:47:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:29.927 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:47:30.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:29 smithi038 ceph-mon[93172]: pgmap v41: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-26T04:47:30.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:29 smithi038 ceph-mon[36743]: pgmap v41: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-26T04:47:30.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:29 smithi055 ceph-mon[30328]: pgmap v41: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-26T04:47:30.413 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:47:32.002 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:31 smithi038 ceph-mon[93172]: pgmap v42: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-26T04:47:32.002 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:32.003 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:31 smithi038 ceph-mon[36743]: pgmap v42: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-26T04:47:32.003 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:31 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:31 smithi055 ceph-mon[30328]: pgmap v42: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2024-04-26T04:47:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:31 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:33.111 INFO:teuthology.orchestra.run.smithi038.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:33.132 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:33.133 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:33 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:34 smithi055 ceph-mon[30328]: pgmap v43: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-04-26T04:47:34.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:34 smithi055 ceph-mon[30328]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:47:34.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:34 smithi055 ceph-mon[30328]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:34.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:34 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:34.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[93172]: pgmap v43: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-04-26T04:47:34.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[93172]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:47:34.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[93172]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:34.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:34.375 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[36743]: pgmap v43: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-04-26T04:47:34.375 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[36743]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:47:34.376 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[36743]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:34.376 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:34 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:34.376 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-26T04:47:35.557 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:47:36.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:36 smithi055 ceph-mon[30328]: pgmap v44: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2024-04-26T04:47:36.422 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:36 smithi038 ceph-mon[93172]: pgmap v44: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2024-04-26T04:47:36.422 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:36 smithi038 ceph-mon[36743]: pgmap v44: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 929 op/s 2024-04-26T04:47:37.179 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:47:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:47:36] "GET /metrics HTTP/1.1" 200 36123 "" "Prometheus/2.43.0" 2024-04-26T04:47:38.159 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:38 smithi038 ceph-mon[93172]: pgmap v45: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-04-26T04:47:38.159 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:38 smithi038 ceph-mon[36743]: pgmap v45: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-04-26T04:47:38.170 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:38 smithi055 ceph-mon[30328]: pgmap v45: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-04-26T04:47:38.371 INFO:teuthology.orchestra.run.smithi038.stdout:wait for majority of mons to upgrade 2024-04-26T04:47:39.022 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-26T04:47:39.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:47:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:47:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:40.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:39 smithi038 ceph-mon[93172]: pgmap v46: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-26T04:47:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:39 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:40.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:39 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:40.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:39 smithi038 ceph-mon[36743]: pgmap v46: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-26T04:47:40.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:39 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:40.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:39 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:40.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:39 smithi055 ceph-mon[30328]: pgmap v46: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-04-26T04:47:40.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:39 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:40.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:39 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:41.454 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:47:42.223 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:41 smithi038 ceph-mon[93172]: pgmap v47: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-26T04:47:42.223 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:41 smithi038 ceph-mon[36743]: pgmap v47: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-26T04:47:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:41 smithi055 ceph-mon[30328]: pgmap v47: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-26T04:47:43.827 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:43.827 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:43.827 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:43 smithi038 ceph-mon[93172]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-04-26T04:47:43.827 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:43 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:43.827 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:43 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:43.827 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:43 smithi038 ceph-mon[36743]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-04-26T04:47:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:43 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:43 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:43 smithi055 ceph-mon[30328]: pgmap v48: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-04-26T04:47:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:47:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: pgmap v49: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-04-26T04:47:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:47:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:47:45.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:44 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:47:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:47:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: pgmap v49: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: pgmap v49: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:47:45.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:44 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-26T04:47:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:45 smithi038 ceph-mon[93172]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:47:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:45 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3732905848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:45.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:45 smithi038 ceph-mon[36743]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:47:45.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:45 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3732905848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:46.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:45 smithi055 ceph-mon[30328]: Upgrade: It appears safe to stop mon.c 2024-04-26T04:47:46.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:45 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3732905848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:46 smithi055 ceph-mon[30328]: pgmap v50: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:47:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:46 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:46 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:47:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:46 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:47:47.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:46 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:47.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:47:46] "GET /metrics HTTP/1.1" 200 36114 "" "Prometheus/2.43.0" 2024-04-26T04:47:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[93172]: pgmap v50: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:47:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:47:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:47:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[36743]: pgmap v50: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-04-26T04:47:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:47:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:47:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:47:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:46 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:47:47.794 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:47 smithi038 ceph-mon[93172]: Upgrade: Updating mon.c 2024-04-26T04:47:47.795 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:47 smithi038 ceph-mon[93172]: Deploying daemon mon.c on smithi038 2024-04-26T04:47:47.795 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:47 smithi055 ceph-mon[30328]: Upgrade: Updating mon.c 2024-04-26T04:47:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:47 smithi055 ceph-mon[30328]: Deploying daemon mon.c on smithi038 2024-04-26T04:47:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:47 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:48.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:47 smithi038 ceph-mon[36743]: Upgrade: Updating mon.c 2024-04-26T04:47:48.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:47 smithi038 ceph-mon[36743]: Deploying daemon mon.c on smithi038 2024-04-26T04:47:48.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:47 smithi038 ceph-mon[36743]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:47:48.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:48 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3733082248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:48.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:48 smithi038 ceph-mon[93172]: pgmap v51: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 268 B/s wr, 1.03k op/s 2024-04-26T04:47:48.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:48 smithi038 ceph-mon[36743]: from='client.? 172.21.15.38:0/3733082248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:48.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:48 smithi038 ceph-mon[36743]: pgmap v51: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 268 B/s wr, 1.03k op/s 2024-04-26T04:47:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:48 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3733082248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:47:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:48 smithi055 ceph-mon[30328]: pgmap v51: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 268 B/s wr, 1.03k op/s 2024-04-26T04:47:49.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:47:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:47:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:49 smithi038 systemd[1]: Stopping Ceph mon.c for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:47:49.842 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c[36739]: 2024-04-26T04:47:49.532+0000 7ffb9bfe8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:47:49.842 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c[36739]: 2024-04-26T04:47:49.532+0000 7ffb9bfe8700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-26T04:47:49.842 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:49 smithi038 podman[96754]: 2024-04-26 04:47:49.655131324 +0000 UTC m=+0.274113498 container died 96c89d4820732361d5d4c472d4b2b5f35c1f80d87a4d3d088fda8596463c96c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:47:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:50 smithi038 podman[96754]: 2024-04-26 04:47:50.033732 +0000 UTC m=+0.652714179 container cleanup 96c89d4820732361d5d4c472d4b2b5f35c1f80d87a4d3d088fda8596463c96c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c) 2024-04-26T04:47:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:50 smithi038 bash[96754]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c 2024-04-26T04:47:50.881 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:50 smithi038 podman[96765]: 2024-04-26 04:47:50.602794387 +0000 UTC m=+0.946924485 container remove 96c89d4820732361d5d4c472d4b2b5f35c1f80d87a4d3d088fda8596463c96c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:47:51.220 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:50 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-26T04:47:51.220 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:50 smithi038 systemd[1]: Stopped Ceph mon.c for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:47:51.220 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:50 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.c.service: Consumed 10.567s CPU time. 2024-04-26T04:47:51.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:51 smithi038 systemd[1]: Starting Ceph mon.c for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:47:51.934 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:51 smithi038 podman[96906]: 2024-04-26 04:47:51.548151343 +0000 UTC m=+0.044340790 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:47:51.934 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:51 smithi038 podman[96906]: 2024-04-26 04:47:51.712411797 +0000 UTC m=+0.208601237 container create 7e20996682cea48c78cf2bad1aadbcae5df13439dcfaea319dc69c6b0e0c5a72 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:47:52.230 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 podman[96906]: 2024-04-26 04:47:52.008199226 +0000 UTC m=+0.504388664 container init 7e20996682cea48c78cf2bad1aadbcae5df13439dcfaea319dc69c6b0e0c5a72 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 podman[96906]: 2024-04-26 04:47:52.015221069 +0000 UTC m=+0.511410510 container start 7e20996682cea48c78cf2bad1aadbcae5df13439dcfaea319dc69c6b0e0c5a72 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: pidfile_write: ignore empty --pid-file 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: load: jerasure load: lrc 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Git sha 0 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: DB SUMMARY 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: DB Session ID: MK22DXX2QSJZXW9GAIEI 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:47:52.231 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1082 Bytes 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 8210119 ; 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.env: 0x563972e54c60 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.info_log: 0x563973ad3aa0 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:47:52.232 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.statistics: (nil) 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.use_fsync: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:47:52.233 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.db_log_dir: 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.wal_dir: 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.write_buffer_manager: 0x563973aa61e0 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:47:52.234 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.unordered_write: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.row_cache: None 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.wal_filter: None 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:47:52.235 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.wal_compression: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:47:52.236 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_open_files: -1 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:47:52.237 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Compression algorithms supported: 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kZSTD supported: 0 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-26T04:47:52.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.merge_operator: 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_filter: None 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563973ad3ba0) 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: index_type: 0 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: data_block_index_type: 0 2024-04-26T04:47:52.239 INFO:journalctl@ceph.mon.c.smithi038.stdout: index_shortening: 1 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: checksum: 4 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: no_block_cache: 0 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_cache: 0x563973ad7090 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_cache_options: 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: capacity : 536870912 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: num_shard_bits : 4 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: strict_capacity_limit : 0 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_cache_compressed: (nil) 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: persistent_cache: (nil) 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_size: 4096 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_size_deviation: 10 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_restart_interval: 16 2024-04-26T04:47:52.240 INFO:journalctl@ceph.mon.c.smithi038.stdout: index_block_restart_interval: 1 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: metadata_block_size: 4096 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: partition_filters: 0 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: use_delta_encoding: 1 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: filter_policy: bloomfilter 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: whole_key_filtering: 1 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: verify_compression: 0 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: format_version: 5 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: enable_index_compression: 1 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: block_align: 0 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: max_auto_readahead_size: 262144 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: prepopulate_block_cache: 0 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:47:52.241 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression: NoCompression 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.num_levels: 7 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:47:52.242 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:47:52.243 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:47:52.244 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:47:52.245 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:47:52.246 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.ttl: 2592000 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:47:52.247 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12662, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T04:47:52.248 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6f9f9ada-87b0-4a0c-9660-586958bfaec8 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106872077275, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 bash[96906]: 7e20996682cea48c78cf2bad1aadbcae5df13439dcfaea319dc69c6b0e0c5a72 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106872122233, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5058146, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12659, "largest_seqno": 15451, "table_properties": {"data_size": 5046417, "index_size": 7508, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3205, "raw_key_size": 27036, "raw_average_key_size": 21, "raw_value_size": 5021249, "raw_average_value_size": 4000, "num_data_blocks": 338, "num_entries": 1255, "num_filter_entries": 1255, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106872, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6f9f9ada-87b0-4a0c-9660-586958bfaec8", "db_session_id": "MK22DXX2QSJZXW9GAIEI", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106872122589, "job": 1, "event": "recovery_finished"} 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 systemd[1]: Started Ceph mon.c for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563973bd2000 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: DB pointer 0x563973bbc000 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: starting mon.c rank 2 at public addrs [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] at bind addrs [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:47:52.249 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???) e3 preinit fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).mds e9 new map 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).mds e9 print_map 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: e9 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: legacy client fscid: 1 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: Filesystem 'foo' (1) 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: fs_name foo 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: epoch 8 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: created 2024-04-26T04:39:56.305501+0000 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: modified 2024-04-26T04:40:12.113527+0000 2024-04-26T04:47:52.250 INFO:journalctl@ceph.mon.c.smithi038.stdout: tableserver 0 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: root 0 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: session_timeout 60 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: session_autoclose 300 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: max_file_size 1099511627776 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: max_xattr_size 65536 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: required_client_features {} 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: last_failure 0 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: last_failure_osd_epoch 0 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: max_mds 1 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: in 0 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: up {0=24554} 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: failed 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: damaged 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: stopped 2024-04-26T04:47:52.251 INFO:journalctl@ceph.mon.c.smithi038.stdout: data_pools [4] 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: metadata_pool 3 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: inline_data disabled 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: balancer 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: bal_rank_mask -1 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: standby_count_wanted 1 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: [mds.foo.smithi038.qsffiu{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: Standby daemons: 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout: [mds.foo.smithi055.cuaegn{-1:24532} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:47:52.252 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:52 smithi038 ceph-mon[96929]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: ** DB Stats ** 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: Uptime(secs): 0.1 total, 0.1 interval 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:47:52.253 INFO:journalctl@ceph.mon.c.smithi038.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: L0 1/0 4.82 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 303.4 0.02 0.00 1 0.016 0 0 0.0 0.0 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: L6 1/0 5.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: Sum 2/0 10.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 303.4 0.02 0.00 1 0.016 0 0 0.0 0.0 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 303.4 0.02 0.00 1 0.016 0 0 0.0 0.0 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: ** Compaction Stats [default] ** 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 303.4 0.02 0.00 1 0.016 0 0 0.0 0.0 2024-04-26T04:47:52.254 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Uptime(secs): 0.1 total, 0.1 interval 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Cumulative compaction: 0.00 GB write, 69.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Interval compaction: 0.00 GB write, 69.16 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Block cache BinnedLRUCache@0x563973ad7090#2 capacity: 512.00 MB usage: 303.20 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: Block cache entry stats(count,size,portion): DataBlock(19,266.73 KB,0.0508755%) FilterBlock(2,12.31 KB,0.00234842%) IndexBlock(2,24.16 KB,0.00460744%) Misc(1,0.00 KB,0%) 2024-04-26T04:47:52.255 INFO:journalctl@ceph.mon.c.smithi038.stdout: 2024-04-26T04:47:52.256 INFO:journalctl@ceph.mon.c.smithi038.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:47:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:47:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:47:56] "GET /metrics HTTP/1.1" 200 36114 "" "Prometheus/2.43.0" 2024-04-26T04:47:57.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:57 smithi038 ceph-mon[93172]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T04:47:57.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:57 smithi055 ceph-mon[30328]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T04:47:58.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: mgrmap e39: y(active, since 92s), standbys: x 2024-04-26T04:47:58.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: mon.c calling monitor election 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: mon.b calling monitor election 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: mon.a calling monitor election 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: monmap epoch 3 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: last_changed 2024-04-26T04:34:38.393611+0000 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: min_mon_release 17 (quincy) 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: election_strategy: 1 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: mgrmap e39: y(active, since 101s), standbys: x 2024-04-26T04:47:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[96929]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: mon.c calling monitor election 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: mon.b calling monitor election 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: mon.a calling monitor election 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: monmap epoch 3 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: last_changed 2024-04-26T04:34:38.393611+0000 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: min_mon_release 17 (quincy) 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: election_strategy: 1 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:47:58.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:47:58.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:47:58.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:47:58.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:47:58.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: mgrmap e39: y(active, since 101s), standbys: x 2024-04-26T04:47:58.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:58 smithi038 ceph-mon[93172]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:47:58.856 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:47:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:47:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: mon.c calling monitor election 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: mon.b calling monitor election 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: mon.a calling monitor election 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: monmap epoch 3 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: last_changed 2024-04-26T04:34:38.393611+0000 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: min_mon_release 17 (quincy) 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: election_strategy: 1 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:47:58.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: mgrmap e39: y(active, since 101s), standbys: x 2024-04-26T04:47:58.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:58 smithi055 ceph-mon[30328]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:47:59.510 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T04:47:59.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:59 smithi038 ceph-mon[93172]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:47:59.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:59 smithi038 ceph-mon[93172]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:47:59.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:47:59 smithi038 ceph-mon[93172]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:47:59.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:59 smithi038 ceph-mon[96929]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:47:59.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:59 smithi038 ceph-mon[96929]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:47:59.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:47:59 smithi038 ceph-mon[96929]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:47:59.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:59 smithi055 ceph-mon[30328]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:47:59.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:59 smithi055 ceph-mon[30328]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:47:59.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:47:59 smithi055 ceph-mon[30328]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:48:00.186 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:48:00.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/428457755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:00 smithi038 ceph-mon[93172]: pgmap v57: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-26T04:48:00.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/428457755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:00 smithi038 ceph-mon[96929]: pgmap v57: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-26T04:48:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:00 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/428457755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:48:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:00 smithi055 ceph-mon[30328]: pgmap v57: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-26T04:48:01.176 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:48:02.065 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:02.065 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:02.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:01 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:02.374 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:02 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:48:02.170+0000 7fa1d4887700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-26T04:48:03.332 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:03 smithi038 ceph-mon[93172]: pgmap v58: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-26T04:48:03.332 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:03 smithi038 ceph-mon[96929]: pgmap v58: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-26T04:48:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:03 smithi055 ceph-mon[30328]: pgmap v58: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-26T04:48:05.152 INFO:teuthology.orchestra.run.smithi038.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:48:05.152 INFO:teuthology.orchestra.run.smithi038.stdout:alertmanager.a smithi038 *:9093,9094 running (3m) 22s ago 9m 27.5M - 0.25.0 c8568f914cd2 bd09134fac6b 2024-04-26T04:48:05.152 INFO:teuthology.orchestra.run.smithi038.stdout:grafana.a smithi055 *:3000 error 26s ago 8m - - 2024-04-26T04:48:05.152 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi038.qsffiu smithi038 running (7m) 22s ago 7m 16.4M - 17.2.7-902-gc61472c8 0de407c1b3af 841ef3f1e866 2024-04-26T04:48:05.152 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi055.cuaegn smithi055 running (7m) 26s ago 7m 13.3M - 17.2.7-902-gc61472c8 0de407c1b3af 030d9ab50e92 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.x smithi055 *:8443,9283,8765 running (2m) 26s ago 13m 449M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f26c41ca1326 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.y smithi038 *:8443,9283,8765 running (3m) 22s ago 14m 499M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4c95c5f5ae02 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:mon.a smithi038 running (2m) 22s ago 15m 38.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c46affca8e4e 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:mon.b smithi055 running (13m) 26s ago 13m 52.2M 2048M 17.2.7-902-gc61472c8 0de407c1b3af e2ab87d945f4 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:mon.c smithi038 starting - - - 2048M 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.a smithi038 *:9100 running (3m) 22s ago 9m 15.5M - 1.5.0 0da6a335fe13 f3d04e42cf8f 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.b smithi055 *:9100 running (3m) 26s ago 9m 15.5M - 1.5.0 0da6a335fe13 9087143d9886 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.0 smithi038 running (12m) 22s ago 12m 380M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 7a0711df145c 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.1 smithi038 running (12m) 22s ago 12m 329M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 54e6c9519bdd 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.2 smithi038 running (12m) 22s ago 12m 457M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 05094c28a4cd 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.3 smithi038 running (11m) 22s ago 11m 473M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 61ce04224439 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.4 smithi055 running (11m) 26s ago 11m 401M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 40ece9b7ce4f 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.5 smithi055 running (10m) 26s ago 10m 435M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 50fe68fbb905 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.6 smithi055 running (10m) 26s ago 10m 298M 2481M 17.2.7-902-gc61472c8 0de407c1b3af bc76cfe8fbcd 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:osd.7 smithi055 running (10m) 26s ago 10m 376M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 6ca051d18a67 2024-04-26T04:48:05.153 INFO:teuthology.orchestra.run.smithi038.stdout:prometheus.a smithi055 *:9095 running (2m) 26s ago 9m 59.9M - 2.43.0 a07b618ecd1d c40205ac5452 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:05 smithi038 ceph-mon[93172]: pgmap v59: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:48:05.525 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:05 smithi038 ceph-mon[96929]: pgmap v59: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:48:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:05 smithi055 ceph-mon[30328]: pgmap v59: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:48:06.454 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-mon[96929]: from='client.25285 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:06.454 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:06.454 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:06.454 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-mon[93172]: from='client.25285 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:06.454 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:06.454 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:06.497 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:48:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:06 smithi055 ceph-mon[30328]: from='client.25285 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:06 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:06 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:07.042 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:48:07.189 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:48:06] "GET /metrics HTTP/1.1" 200 36119 "" "Prometheus/2.43.0" 2024-04-26T04:48:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:07 smithi038 ceph-mon[93172]: pgmap v60: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-04-26T04:48:07.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:07 smithi038 ceph-mon[96929]: pgmap v60: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-04-26T04:48:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:07 smithi055 ceph-mon[30328]: pgmap v60: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-04-26T04:48:09.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:48:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:48:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:09.482 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:09 smithi055 ceph-mon[30328]: pgmap v61: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-04-26T04:48:09.483 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:09 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:09.483 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:09 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:09.523 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:09 smithi038 ceph-mon[93172]: pgmap v61: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-04-26T04:48:09.523 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:09.523 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:09.523 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:09 smithi038 ceph-mon[96929]: pgmap v61: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2024-04-26T04:48:09.523 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:09.523 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:10.761 INFO:teuthology.orchestra.run.smithi038.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:11.599 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:11 smithi038 ceph-mon[93172]: pgmap v62: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:11.599 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:11 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:11.599 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:11 smithi038 ceph-mon[96929]: pgmap v62: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:11.600 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:11 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:11 smithi055 ceph-mon[30328]: pgmap v62: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:11 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:12.216 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:48:12.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:12 smithi038 ceph-mon[93172]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:12.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:12 smithi038 ceph-mon[93172]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:12.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:12 smithi038 ceph-mon[96929]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:12.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:12 smithi038 ceph-mon[96929]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:12 smithi055 ceph-mon[30328]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:48:12.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:12 smithi055 ceph-mon[30328]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:48:13.349 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:48:13.488 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:13 smithi038 ceph-mon[93172]: pgmap v63: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:13.488 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:13 smithi038 ceph-mon[96929]: pgmap v63: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:13.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:13 smithi055 ceph-mon[30328]: pgmap v63: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:14 smithi055 ceph-mon[30328]: pgmap v64: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-04-26T04:48:14.656 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:14 smithi038 ceph-mon[93172]: pgmap v64: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-04-26T04:48:14.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:14 smithi038 ceph-mon[96929]: pgmap v64: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-04-26T04:48:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:48:16] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.43.0" 2024-04-26T04:48:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: pgmap v65: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:17 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: pgmap v65: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: pgmap v65: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2024-04-26T04:48:17.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:48:17.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:18 smithi055 ceph-mon[30328]: pgmap v66: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-26T04:48:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:18 smithi055 ceph-mon[30328]: pgmap v67: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s 2024-04-26T04:48:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:18 smithi055 ceph-mon[30328]: pgmap v68: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-04-26T04:48:18.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:18 smithi055 ceph-mon[30328]: pgmap v69: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-04-26T04:48:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[93172]: pgmap v66: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-26T04:48:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[93172]: pgmap v67: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s 2024-04-26T04:48:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[93172]: pgmap v68: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-04-26T04:48:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[93172]: pgmap v69: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-04-26T04:48:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[96929]: pgmap v66: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-26T04:48:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[96929]: pgmap v67: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s 2024-04-26T04:48:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[96929]: pgmap v68: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2024-04-26T04:48:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:18 smithi038 ceph-mon[96929]: pgmap v69: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-04-26T04:48:19.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:48:18 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:48:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:20 smithi055 ceph-mon[30328]: pgmap v70: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-26T04:48:20.526 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:20 smithi038 ceph-mon[93172]: pgmap v70: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-26T04:48:20.526 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:20 smithi038 ceph-mon[96929]: pgmap v70: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-26T04:48:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:22 smithi055 ceph-mon[30328]: pgmap v71: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-26T04:48:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:22 smithi038 ceph-mon[93172]: pgmap v71: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-26T04:48:22.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:22 smithi038 ceph-mon[96929]: pgmap v71: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-26T04:48:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:24 smithi055 ceph-mon[30328]: pgmap v72: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-04-26T04:48:24.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:24 smithi038 ceph-mon[93172]: pgmap v72: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-04-26T04:48:24.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:24 smithi038 ceph-mon[96929]: pgmap v72: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-04-26T04:48:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:26 smithi055 ceph-mon[30328]: pgmap v73: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:48:26.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:26 smithi038 ceph-mon[93172]: pgmap v73: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:48:26.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:26 smithi038 ceph-mon[96929]: pgmap v73: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:48:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:48:26] "GET /metrics HTTP/1.1" 200 36117 "" "Prometheus/2.43.0" 2024-04-26T04:48:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:28 smithi055 ceph-mon[30328]: pgmap v74: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2024-04-26T04:48:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:28 smithi038 ceph-mon[93172]: pgmap v74: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2024-04-26T04:48:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:28 smithi038 ceph-mon[96929]: pgmap v74: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2024-04-26T04:48:29.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:48:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:48:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:30 smithi055 ceph-mon[30328]: pgmap v75: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-26T04:48:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:30 smithi038 ceph-mon[93172]: pgmap v75: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-26T04:48:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:30 smithi038 ceph-mon[96929]: pgmap v75: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-26T04:48:32.589 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:32 smithi038 ceph-mon[93172]: pgmap v76: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-26T04:48:32.589 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:32.590 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:32 smithi038 ceph-mon[96929]: pgmap v76: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-26T04:48:32.590 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:32 smithi055 ceph-mon[30328]: pgmap v76: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-26T04:48:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:33 smithi038 ceph-mon[96929]: pgmap v77: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-04-26T04:48:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:33 smithi038 ceph-mon[93172]: pgmap v77: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-04-26T04:48:33.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:33 smithi055 ceph-mon[30328]: pgmap v77: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-04-26T04:48:36.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:35 smithi055 ceph-mon[30328]: pgmap v78: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:48:36.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:35 smithi038 ceph-mon[96929]: pgmap v78: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:48:36.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:35 smithi038 ceph-mon[93172]: pgmap v78: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-04-26T04:48:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:48:36] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.43.0" 2024-04-26T04:48:38.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:37 smithi055 ceph-mon[30328]: pgmap v79: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-26T04:48:38.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:37 smithi038 ceph-mon[96929]: pgmap v79: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-26T04:48:38.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:37 smithi038 ceph-mon[93172]: pgmap v79: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-04-26T04:48:39.007 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:48:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:48:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:40.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:39 smithi055 ceph-mon[30328]: pgmap v80: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-26T04:48:40.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:39 smithi038 ceph-mon[93172]: pgmap v80: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-26T04:48:40.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:39 smithi038 ceph-mon[96929]: pgmap v80: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-26T04:48:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:41 smithi055 ceph-mon[30328]: pgmap v81: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-26T04:48:42.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:41 smithi038 ceph-mon[93172]: pgmap v81: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-26T04:48:42.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:41 smithi038 ceph-mon[96929]: pgmap v81: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-26T04:48:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:43 smithi055 ceph-mon[30328]: pgmap v82: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2024-04-26T04:48:44.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:43 smithi038 ceph-mon[93172]: pgmap v82: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2024-04-26T04:48:44.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:43 smithi038 ceph-mon[96929]: pgmap v82: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2024-04-26T04:48:46.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:45 smithi055 ceph-mon[30328]: pgmap v83: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-04-26T04:48:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:45 smithi038 ceph-mon[93172]: pgmap v83: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-04-26T04:48:46.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:45 smithi038 ceph-mon[96929]: pgmap v83: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-04-26T04:48:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:46 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:47.191 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:47.192 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:48:47.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:47 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:48:47] "GET /metrics HTTP/1.1" 200 36130 "" "Prometheus/2.43.0" 2024-04-26T04:48:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:47 smithi055 ceph-mon[30328]: pgmap v84: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-04-26T04:48:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:47 smithi038 ceph-mon[93172]: pgmap v84: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-04-26T04:48:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:47 smithi038 ceph-mon[96929]: pgmap v84: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-04-26T04:48:49.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:48:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:48:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:48:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:49 smithi055 ceph-mon[30328]: pgmap v85: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-26T04:48:50.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:49 smithi038 ceph-mon[93172]: pgmap v85: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-26T04:48:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:49 smithi038 ceph-mon[96929]: pgmap v85: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-26T04:48:52.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:51 smithi055 ceph-mon[30328]: pgmap v86: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-26T04:48:52.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:51 smithi038 ceph-mon[93172]: pgmap v86: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-26T04:48:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:51 smithi038 ceph-mon[96929]: pgmap v86: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-26T04:48:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:53 smithi055 ceph-mon[30328]: pgmap v87: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2024-04-26T04:48:54.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:53 smithi038 ceph-mon[93172]: pgmap v87: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2024-04-26T04:48:54.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:53 smithi038 ceph-mon[96929]: pgmap v87: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 841 op/s 2024-04-26T04:48:56.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:55 smithi055 ceph-mon[30328]: pgmap v88: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-04-26T04:48:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:55 smithi038 ceph-mon[93172]: pgmap v88: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-04-26T04:48:56.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:55 smithi038 ceph-mon[96929]: pgmap v88: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-04-26T04:48:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:48:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:48:56] "GET /metrics HTTP/1.1" 200 36123 "" "Prometheus/2.43.0" 2024-04-26T04:48:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:57 smithi055 ceph-mon[30328]: pgmap v89: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-04-26T04:48:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:57 smithi038 ceph-mon[93172]: pgmap v89: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-04-26T04:48:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:57 smithi038 ceph-mon[96929]: pgmap v89: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-04-26T04:48:59.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:48:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:48:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:49:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:48:59 smithi038 ceph-mon[93172]: pgmap v90: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-26T04:49:00.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:48:59 smithi038 ceph-mon[96929]: pgmap v90: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-26T04:49:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:48:59 smithi055 ceph-mon[30328]: pgmap v90: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-26T04:49:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:01 smithi038 ceph-mon[93172]: pgmap v91: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-26T04:49:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:02.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:01 smithi038 ceph-mon[96929]: pgmap v91: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-26T04:49:02.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:01 smithi055 ceph-mon[30328]: pgmap v91: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-26T04:49:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:01 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:03 smithi038 ceph-mon[93172]: pgmap v92: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-04-26T04:49:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:03 smithi038 ceph-mon[96929]: pgmap v92: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-04-26T04:49:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:03 smithi055 ceph-mon[30328]: pgmap v92: 89 pgs: 89 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 800 op/s 2024-04-26T04:49:06.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:05 smithi038 ceph-mon[93172]: pgmap v93: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-04-26T04:49:06.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:05 smithi038 ceph-mon[96929]: pgmap v93: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-04-26T04:49:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:05 smithi055 ceph-mon[30328]: pgmap v93: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 980 op/s 2024-04-26T04:49:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:49:06] "GET /metrics HTTP/1.1" 200 36122 "" "Prometheus/2.43.0" 2024-04-26T04:49:08.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:07 smithi038 ceph-mon[93172]: pgmap v94: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-26T04:49:08.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:07 smithi038 ceph-mon[96929]: pgmap v94: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-26T04:49:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:07 smithi055 ceph-mon[30328]: pgmap v94: 89 pgs: 89 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-26T04:49:09.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 04:49:08 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T04:49:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi055.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)\", cluster_addr=\"172.21.15.38\", device_class=\"ssd\", hostname=\"smithi038\", instance=\"smithi038:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.38\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-26T04:49:10.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:09 smithi038 ceph-mon[93172]: pgmap v95: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-26T04:49:10.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:09 smithi038 ceph-mon[96929]: pgmap v95: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-26T04:49:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:09 smithi055 ceph-mon[30328]: pgmap v95: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-26T04:49:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:11 smithi038 ceph-mon[93172]: pgmap v96: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-04-26T04:49:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:11 smithi038 ceph-mon[96929]: pgmap v96: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-04-26T04:49:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:11 smithi055 ceph-mon[30328]: pgmap v96: 89 pgs: 89 active+clean; 1.6 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2024-04-26T04:49:14.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:13 smithi038 ceph-mon[93172]: pgmap v97: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-04-26T04:49:14.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:13 smithi038 ceph-mon[96929]: pgmap v97: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-04-26T04:49:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:13 smithi055 ceph-mon[30328]: pgmap v97: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2024-04-26T04:49:16.017 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:49:16.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:15 smithi038 ceph-mon[93172]: pgmap v98: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:49:16.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:15 smithi038 ceph-mon[96929]: pgmap v98: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:49:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:15 smithi055 ceph-mon[30328]: pgmap v98: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-26T04:49:16.495 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:17.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:49:16] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-04-26T04:49:17.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:16 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:17.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:16 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:17.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:16 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:17.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:16 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:16 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:16 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:18.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:17 smithi038 ceph-mon[93172]: pgmap v99: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-04-26T04:49:18.209 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:17 smithi038 ceph-mon[96929]: pgmap v99: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-04-26T04:49:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:17 smithi055 ceph-mon[30328]: pgmap v99: 89 pgs: 89 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-04-26T04:49:19.006 INFO:teuthology.orchestra.run.smithi038.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:20.178 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[93172]: pgmap v100: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-26T04:49:20.178 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[93172]: from='client.25294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:20.178 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[93172]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:20.178 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:20.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[96929]: pgmap v100: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-26T04:49:20.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[96929]: from='client.25294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:20.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[96929]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:20.179 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:20.225 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for all mons to upgrade' 2024-04-26T04:49:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:20 smithi055 ceph-mon[30328]: pgmap v100: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-26T04:49:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:20 smithi055 ceph-mon[30328]: from='client.25294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:20 smithi055 ceph-mon[30328]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:20 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:21.630 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:22.152 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:22 smithi038 ceph-mon[96929]: pgmap v101: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-26T04:49:22.152 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:22 smithi038 ceph-mon[93172]: pgmap v101: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-26T04:49:22.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:22 smithi055 ceph-mon[30328]: pgmap v101: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-26T04:49:23.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:23 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:23.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:23 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:23.453 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:23.453 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:23.453 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:23.454 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:23.919 INFO:teuthology.orchestra.run.smithi038.stdout:wait for all mons to upgrade 2024-04-26T04:49:24.382 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:24 smithi038 ceph-mon[96929]: pgmap v102: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-26T04:49:24.383 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:24 smithi038 ceph-mon[93172]: pgmap v102: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-26T04:49:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:24 smithi055 ceph-mon[30328]: pgmap v102: 89 pgs: 89 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-04-26T04:49:24.845 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-26T04:49:25.423 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:26.164 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:26 smithi038 ceph-mon[93172]: pgmap v103: 89 pgs: 89 active+clean; 856 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-04-26T04:49:26.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:26 smithi038 ceph-mon[96929]: pgmap v103: 89 pgs: 89 active+clean; 856 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-04-26T04:49:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:26 smithi055 ceph-mon[30328]: pgmap v103: 89 pgs: 89 active+clean; 856 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-04-26T04:49:27.201 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:49:26] "GET /metrics HTTP/1.1" 200 36121 "" "Prometheus/2.43.0" 2024-04-26T04:49:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:28 smithi038 ceph-mon[93172]: pgmap v104: 89 pgs: 89 active+clean; 856 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2024-04-26T04:49:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:28 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:28 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:28 smithi038 ceph-mon[96929]: pgmap v104: 89 pgs: 89 active+clean; 856 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2024-04-26T04:49:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:28 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:28.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:28 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:28 smithi055 ceph-mon[30328]: pgmap v104: 89 pgs: 89 active+clean; 856 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2024-04-26T04:49:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:28 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:28 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/843330180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/843330180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:49:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:49:29.519 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:29.519 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:29.519 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:29.520 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/843330180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:29.520 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:29.520 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:49:29.520 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-26T04:49:29.520 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:29 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-26T04:49:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:30 smithi038 ceph-mon[93172]: pgmap v105: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:49:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:30 smithi038 ceph-mon[93172]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:49:30.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:30 smithi038 ceph-mon[96929]: pgmap v105: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:49:30.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:30 smithi038 ceph-mon[96929]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:49:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:30 smithi055 ceph-mon[30328]: pgmap v105: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-26T04:49:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:30 smithi055 ceph-mon[30328]: Upgrade: It appears safe to stop mon.b 2024-04-26T04:49:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: pgmap v106: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-26T04:49:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3833153918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: pgmap v106: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3833153918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: pgmap v106: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-26T04:49:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3833153918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:32.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:32.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:32.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:32 smithi055 ceph-mon[30328]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:33 smithi038 ceph-mon[93172]: Upgrade: Updating mon.b 2024-04-26T04:49:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:33 smithi038 ceph-mon[93172]: Deploying daemon mon.b on smithi055 2024-04-26T04:49:33.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:33 smithi038 ceph-mon[96929]: Upgrade: Updating mon.b 2024-04-26T04:49:33.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:33 smithi038 ceph-mon[96929]: Deploying daemon mon.b on smithi055 2024-04-26T04:49:33.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:33 smithi055 ceph-mon[30328]: Upgrade: Updating mon.b 2024-04-26T04:49:33.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:33 smithi055 ceph-mon[30328]: Deploying daemon mon.b on smithi055 2024-04-26T04:49:34.349 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:34 smithi055 ceph-mon[30328]: pgmap v107: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-26T04:49:34.349 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:34 smithi055 ceph-mon[30328]: from='client.? 172.21.15.38:0/3210811259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:34 smithi038 ceph-mon[93172]: pgmap v107: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-26T04:49:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:34 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3210811259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:34 smithi038 ceph-mon[96929]: pgmap v107: 89 pgs: 89 active+clean; 608 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2024-04-26T04:49:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:34 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3210811259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:34 smithi055 systemd[1]: Stopping Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:49:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b[30322]: 2024-04-26T04:49:34.764+0000 7f740bf59700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:49:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b[30322]: 2024-04-26T04:49:34.764+0000 7f740bf59700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-26T04:49:35.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:34 smithi055 podman[86763]: 2024-04-26 04:49:34.984011881 +0000 UTC m=+0.368343412 container died e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-26T04:49:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:35 smithi055 podman[86763]: 2024-04-26 04:49:35.414847819 +0000 UTC m=+0.799179350 container cleanup e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-c61472c) 2024-04-26T04:49:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:35 smithi055 bash[86763]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b 2024-04-26T04:49:36.054 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:35 smithi055 podman[86777]: 2024-04-26 04:49:35.658256653 +0000 UTC m=+0.673201449 container remove e2ab87d945f41f84ece6991eba6f0334e10b9d528fbb60542d569d4bbec49b20 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, org.label-schema.build-date=20240424, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T04:49:36.351 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:36 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-26T04:49:36.351 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:36 smithi055 systemd[1]: Stopped Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:49:36.352 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:36 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.b.service: Consumed 12.829s CPU time. 2024-04-26T04:49:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:36 smithi055 systemd[1]: Starting Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:49:37.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:36 smithi055 podman[86888]: 2024-04-26 04:49:36.53423887 +0000 UTC m=+0.028778246 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:49:36] "GET /metrics HTTP/1.1" 200 36119 "" "Prometheus/2.43.0" 2024-04-26T04:49:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 podman[86888]: 2024-04-26 04:49:37.318713281 +0000 UTC m=+0.813252665 container create 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 podman[86888]: 2024-04-26 04:49:37.613683752 +0000 UTC m=+1.108223098 container init 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 podman[86888]: 2024-04-26 04:49:37.619079106 +0000 UTC m=+1.113618451 container start 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, ceph=True) 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: set uid:gid to 167:167 (ceph:ceph) 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev), process ceph-mon, pid 2 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: pidfile_write: ignore empty --pid-file 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: load: jerasure load: lrc 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: RocksDB version: 7.9.2 2024-04-26T04:49:38.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Git sha 0 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Compile date 2024-04-25 16:43:49 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: DB SUMMARY 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: DB Session ID: B2PNE5O1AUQNYYHTLSCM 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: CURRENT file: CURRENT 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: IDENTITY file: IDENTITY 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1082 Bytes 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9910065 ; 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.error_if_exists: 0 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.create_if_missing: 0 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.paranoid_checks: 1 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.env: 0x55eb16157c60 2024-04-26T04:49:38.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.fs: PosixFileSystem 2024-04-26T04:49:38.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.info_log: 0x55eb18767aa0 2024-04-26T04:49:38.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_file_opening_threads: 16 2024-04-26T04:49:38.109 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.statistics: (nil) 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.use_fsync: 0 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_log_file_size: 0 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.keep_log_file_num: 1000 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.recycle_log_file_num: 0 2024-04-26T04:49:38.110 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.allow_fallocate: 1 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.allow_mmap_reads: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.allow_mmap_writes: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.use_direct_reads: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.create_missing_column_families: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.db_log_dir: 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.wal_dir: 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-26T04:49:38.111 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.advise_random_on_open: 1 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.db_write_buffer_size: 0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.write_buffer_manager: 0x55eb1873a1e0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.rate_limiter: (nil) 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.wal_recovery_mode: 2 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.enable_thread_tracking: 0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.enable_pipelined_write: 0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.unordered_write: 0 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-26T04:49:38.112 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.row_cache: None 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.wal_filter: None 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.allow_ingest_behind: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.two_write_queues: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.manual_wal_flush: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.wal_compression: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.atomic_flush: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.log_readahead_size: 0 2024-04-26T04:49:38.113 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.best_efforts_recovery: 0 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.allow_data_in_errors: 0 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.db_host_id: __hostname__ 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_background_jobs: 2 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_background_compactions: -1 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_subcompactions: 1 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_total_wal_size: 0 2024-04-26T04:49:38.114 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_open_files: -1 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bytes_per_sync: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_readahead_size: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_background_flushes: -1 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Compression algorithms supported: 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kZSTD supported: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kXpressCompression supported: 0 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kLZ4HCCompression supported: 1 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kZlibCompression supported: 1 2024-04-26T04:49:38.115 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kSnappyCompression supported: 1 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kLZ4Compression supported: 1 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: kBZip2Compression supported: 0 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.merge_operator: 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_filter: None 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_filter_factory: None 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.sst_partitioner_factory: None 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-26T04:49:38.116 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eb18767ba0) 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: cache_index_and_filter_blocks: 1 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: pin_top_level_index_and_filter: 1 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: index_type: 0 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: data_block_index_type: 0 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: index_shortening: 1 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: checksum: 4 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: no_block_cache: 0 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache: 0x55eb1876b090 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache_name: BinnedLRUCache 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache_options: 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: capacity : 536870912 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: num_shard_bits : 4 2024-04-26T04:49:38.117 INFO:journalctl@ceph.mon.b.smithi055.stdout: strict_capacity_limit : 0 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: high_pri_pool_ratio: 0.000 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_cache_compressed: (nil) 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: persistent_cache: (nil) 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_size: 4096 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_size_deviation: 10 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_restart_interval: 16 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: index_block_restart_interval: 1 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: metadata_block_size: 4096 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: partition_filters: 0 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: use_delta_encoding: 1 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: filter_policy: bloomfilter 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: whole_key_filtering: 1 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: verify_compression: 0 2024-04-26T04:49:38.118 INFO:journalctl@ceph.mon.b.smithi055.stdout: read_amp_bytes_per_bit: 0 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: format_version: 5 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: enable_index_compression: 1 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: block_align: 0 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: max_auto_readahead_size: 262144 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: prepopulate_block_cache: 0 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: initial_auto_readahead_size: 8192 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout: num_file_reads_for_auto_readahead: 2 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.write_buffer_size: 33554432 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_write_buffer_number: 2 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression: NoCompression 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression: Disabled 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.prefix_extractor: nullptr 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.num_levels: 7 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-26T04:49:38.119 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.level: 32767 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.strategy: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-26T04:49:38.120 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.enabled: false 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.target_file_size_base: 67108864 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-26T04:49:38.121 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-26T04:49:38.122 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-26T04:49:38.122 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-26T04:49:38.122 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-26T04:49:38.122 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-26T04:49:38.123 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-26T04:49:38.123 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-26T04:49:38.123 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-26T04:49:38.124 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-26T04:49:38.124 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.arena_block_size: 1048576 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.disable_auto_compactions: 0 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-26T04:49:38.125 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.table_properties_collectors: 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.inplace_update_support: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.bloom_locality: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.max_successive_merges: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.paranoid_file_checks: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.force_consistency_checks: 1 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.report_bg_io_stats: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.ttl: 2592000 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-26T04:49:38.126 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.enable_blob_files: false 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.min_blob_size: 0 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.blob_file_size: 268435456 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.blob_file_starting_level: 0 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-26T04:49:38.127 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12731, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 681a96e0-f04e-4d79-a00f-b550bb1737e8 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106977663693, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106977701358, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6082826, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12733, "largest_seqno": 16684, "table_properties": {"data_size": 6067069, "index_size": 10576, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4165, "raw_key_size": 35276, "raw_average_key_size": 21, "raw_value_size": 6033540, "raw_average_value_size": 3670, "num_data_blocks": 474, "num_entries": 1644, "num_filter_entries": 1644, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714106977, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "681a96e0-f04e-4d79-a00f-b550bb1737e8", "db_session_id": "B2PNE5O1AUQNYYHTLSCM", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714106977701509, "job": 1, "event": "recovery_finished"} 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55eb18858000 2024-04-26T04:49:38.128 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: DB pointer 0x55eb18846000 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** DB Stats ** 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-26T04:49:38.129 INFO:journalctl@ceph.mon.b.smithi055.stdout: L0 1/0 5.80 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 351.1 0.02 0.00 1 0.017 0 0 0.0 0.0 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: L6 1/0 5.94 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: Sum 2/0 11.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 351.1 0.02 0.00 1 0.017 0 0 0.0 0.0 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 351.1 0.02 0.00 1 0.017 0 0 0.0 0.0 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** Compaction Stats [default] ** 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 351.1 0.02 0.00 1 0.017 0 0 0.0 0.0 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-26T04:49:38.130 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: Cumulative compaction: 0.01 GB write, 117.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: Interval compaction: 0.01 GB write, 117.48 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: Block cache BinnedLRUCache@0x55eb1876b090#2 capacity: 512.00 MB usage: 58.45 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: Block cache entry stats(count,size,portion): DataBlock(3,16.73 KB,0.00319183%) FilterBlock(2,13.56 KB,0.00258684%) IndexBlock(2,28.16 KB,0.00537038%) Misc(1,0.00 KB,0%) 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: starting mon.b rank 1 at public addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] at bind addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???) e3 preinit fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).mds e9 new map 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).mds e9 print_map 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: e9 2024-04-26T04:49:38.131 INFO:journalctl@ceph.mon.b.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: legacy client fscid: 1 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: Filesystem 'foo' (1) 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: fs_name foo 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: epoch 8 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: created 2024-04-26T04:39:56.305501+0000 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: modified 2024-04-26T04:40:12.113527+0000 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: tableserver 0 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: root 0 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: session_timeout 60 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: session_autoclose 300 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: max_file_size 1099511627776 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: max_xattr_size 65536 2024-04-26T04:49:38.132 INFO:journalctl@ceph.mon.b.smithi055.stdout: required_client_features {} 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: last_failure 0 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: last_failure_osd_epoch 0 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: max_mds 1 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: in 0 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: up {0=24554} 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: failed 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: damaged 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: stopped 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: data_pools [4] 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: metadata_pool 3 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: inline_data disabled 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: balancer 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: bal_rank_mask -1 2024-04-26T04:49:38.133 INFO:journalctl@ceph.mon.b.smithi055.stdout: standby_count_wanted 1 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: qdb_cluster leader: 0 members: 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: [mds.foo.smithi038.qsffiu{0:24554} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.38:6834/4163341651,v1:172.21.15.38:6835/4163341651] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: Standby daemons: 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout: [mds.foo.smithi055.cuaegn{-1:24532} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.55:6832/149815301,v1:172.21.15.55:6833/149815301] compat {c=[1],r=[1],i=[7ff]}] 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 ceph-mon[86906]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 bash[86888]: 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 2024-04-26T04:49:38.134 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:37 smithi055 systemd[1]: Started Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:49:39.683 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Removed 127451 objects 2024-04-26T04:49:39.683 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Clean up completed and total clean up time :150.715 2024-04-26T04:49:42.955 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-04-26T04:49:42.955 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-04-26T04:49:42.983 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:49:42.725+0000 7fa1d4887700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-26T04:49:43.235 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:43.605 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:43.605 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: mon.b calling monitor election 2024-04-26T04:49:43.605 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: mon.a calling monitor election 2024-04-26T04:49:43.605 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: pgmap v110: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: pgmap v111: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: monmap epoch 4 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: last_changed 2024-04-26T04:49:38.210145+0000 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: min_mon_release 19 (squid) 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: election_strategy: 1 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:49:43.606 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1/3 mons down, quorum a,b 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: mon.c (rank 2) addr [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] is down (out of quorum) 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:43.607 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: mon.b calling monitor election 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: mon.a calling monitor election 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: pgmap v110: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: pgmap v111: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: monmap epoch 4 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: last_changed 2024-04-26T04:49:38.210145+0000 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: min_mon_release 19 (squid) 2024-04-26T04:49:43.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: election_strategy: 1 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1/3 mons down, quorum a,b 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: mon.c (rank 2) addr [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] is down (out of quorum) 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:43.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:43.819 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T04:49:44.392 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:44 smithi055 ceph-mon[86906]: pgmap v112: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-26T04:49:44.392 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:44 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:44.392 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/662911501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:44.608 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:44 smithi038 ceph-mon[93172]: pgmap v112: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-26T04:49:44.608 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:44 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:44.608 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/662911501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:44.664 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:49:45.140 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:45.398 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: mon.c calling monitor election 2024-04-26T04:49:45.398 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: mon.a calling monitor election 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: mon.b calling monitor election 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: monmap epoch 4 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: last_changed 2024-04-26T04:49:38.210145+0000 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: min_mon_release 19 (squid) 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: election_strategy: 1 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:49:45.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[93172]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.b calling monitor election 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.a calling monitor election 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-26T04:49:45.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: pgmap v110: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: pgmap v111: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: monmap epoch 4 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: last_changed 2024-04-26T04:49:38.210145+0000 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: min_mon_release 19 (squid) 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: election_strategy: 1 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:49:45.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1/3 mons down, quorum a,b 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.c (rank 2) addr [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] is down (out of quorum) 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: pgmap v112: 89 pgs: 89 active+clean; 93 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/662911501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.c calling monitor election 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.a calling monitor election 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.b calling monitor election 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:49:45.402 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: monmap epoch 4 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: last_changed 2024-04-26T04:49:38.210145+0000 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: min_mon_release 19 (squid) 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: election_strategy: 1 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:45.403 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:45 smithi038 ceph-mon[96929]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:49:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: mon.c calling monitor election 2024-04-26T04:49:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: mon.a calling monitor election 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: mon.b calling monitor election 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: monmap epoch 4 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: fsid e22c1684-0385-11ef-bc93-c7b262605968 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: last_changed 2024-04-26T04:49:38.210145+0000 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: created 2024-04-26T04:32:57.061143+0000 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: min_mon_release 19 (squid) 2024-04-26T04:49:45.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: election_strategy: 1 2024-04-26T04:49:45.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: 0: [v2:172.21.15.38:3300/0,v1:172.21.15.38:6789/0] mon.a 2024-04-26T04:49:45.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: 1: [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon.b 2024-04-26T04:49:45.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: 2: [v2:172.21.15.38:3301/0,v1:172.21.15.38:6790/0] mon.c 2024-04-26T04:49:45.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: fsmap foo:1 {0=foo.smithi038.qsffiu=up:active} 1 up:standby 2024-04-26T04:49:45.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: osdmap e82: 8 total, 8 up, 8 in 2024-04-26T04:49:45.608 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: mgrmap e39: y(active, since 3m), standbys: x 2024-04-26T04:49:45.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-26T04:49:45.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T04:49:45.609 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:49:45.610 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:45 smithi055 ceph-mon[86906]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:49:46.335 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_1' removed 2024-04-26T04:49:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:46 smithi055 ceph-mon[86906]: pgmap v113: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-26T04:49:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:46 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/418538598' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:49:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:46 smithi055 ceph-mon[86906]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:49:46.615 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-mon[93172]: pgmap v113: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-26T04:49:46.615 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/418538598' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:49:46.615 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-mon[93172]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:49:46.615 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-mon[96929]: pgmap v113: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-04-26T04:49:46.615 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/418538598' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:49:46.616 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-mon[96929]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:49:46.931 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:49:46] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.43.0" 2024-04-26T04:49:47.388 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-26T04:49:47.389 INFO:tasks.radosbench:Beginning radosbench... 2024-04-26T04:49:47.389 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-04-26T04:49:47.389 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-04-26T04:49:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:47 smithi055 ceph-mon[86906]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:49:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:47 smithi055 ceph-mon[86906]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:49:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:47 smithi055 ceph-mon[86906]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:47.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:47.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[93172]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:49:47.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[93172]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:49:47.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[93172]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:47.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:47.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[96929]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[96929]: osdmap e83: 8 total, 8 up, 8 in 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[96929]: pgmap v115: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:47.666 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:47.921 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:48.150 INFO:teuthology.orchestra.run.smithi038.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:49:48.150 INFO:teuthology.orchestra.run.smithi038.stdout:alertmanager.a smithi038 *:9093,9094 running (5m) 20s ago 10m 26.5M - 0.25.0 c8568f914cd2 bd09134fac6b 2024-04-26T04:49:48.150 INFO:teuthology.orchestra.run.smithi038.stdout:grafana.a smithi055 *:3000 error 0s ago 10m - - 2024-04-26T04:49:48.150 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi038.qsffiu smithi038 running (9m) 20s ago 9m 17.1M - 17.2.7-902-gc61472c8 0de407c1b3af 841ef3f1e866 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi055.cuaegn smithi055 running (9m) 0s ago 9m 13.8M - 17.2.7-902-gc61472c8 0de407c1b3af 030d9ab50e92 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.x smithi055 *:8443,9283,8765 running (4m) 0s ago 14m 451M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f26c41ca1326 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.y smithi038 *:8443,9283,8765 running (4m) 20s ago 16m 502M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4c95c5f5ae02 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:mon.a smithi038 running (3m) 20s ago 16m 43.4M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c46affca8e4e 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:mon.b smithi055 running (11s) 0s ago 15m 26.9M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7a992f3a1025 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:mon.c smithi038 running (116s) 20s ago 15m 33.7M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7e20996682ce 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.a smithi038 *:9100 running (5m) 20s ago 10m 15.3M - 1.5.0 0da6a335fe13 f3d04e42cf8f 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.b smithi055 *:9100 running (5m) 0s ago 10m 15.7M - 1.5.0 0da6a335fe13 9087143d9886 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:osd.0 smithi038 running (14m) 20s ago 14m 391M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 7a0711df145c 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:osd.1 smithi038 running (14m) 20s ago 14m 344M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 54e6c9519bdd 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:osd.2 smithi038 running (13m) 20s ago 13m 476M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 05094c28a4cd 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:osd.3 smithi038 running (13m) 20s ago 13m 493M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 61ce04224439 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:osd.4 smithi055 running (12m) 0s ago 12m 414M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 40ece9b7ce4f 2024-04-26T04:49:48.151 INFO:teuthology.orchestra.run.smithi038.stdout:osd.5 smithi055 running (12m) 0s ago 12m 444M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 50fe68fbb905 2024-04-26T04:49:48.152 INFO:teuthology.orchestra.run.smithi038.stdout:osd.6 smithi055 running (12m) 0s ago 12m 313M 2481M 17.2.7-902-gc61472c8 0de407c1b3af bc76cfe8fbcd 2024-04-26T04:49:48.152 INFO:teuthology.orchestra.run.smithi038.stdout:osd.7 smithi055 running (11m) 0s ago 11m 391M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 6ca051d18a67 2024-04-26T04:49:48.152 INFO:teuthology.orchestra.run.smithi038.stdout:prometheus.a smithi055 *:9095 running (4m) 0s ago 11m 64.2M - 2.43.0 a07b618ecd1d c40205ac5452 2024-04-26T04:49:48.754 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:49:48.945 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:48 smithi038 ceph-mon[93172]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:48.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:48.946 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:48.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:48 smithi038 ceph-mon[96929]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:48.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:48.946 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:49.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:48 smithi055 ceph-mon[86906]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:49.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:49.024 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:49.200 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:49.944 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:49 smithi038 ceph-mon[93172]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:49.944 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:49 smithi038 ceph-mon[96929]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:50.033 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:49 smithi055 ceph-mon[86906]: pgmap v116: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:51.754 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_2' created 2024-04-26T04:49:51.843 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:51 smithi055 ceph-mon[86906]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:51.844 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:51 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4191329433' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-26T04:49:51.939 INFO:teuthology.orchestra.run.smithi038.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:51.961 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:51 smithi038 ceph-mon[93172]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:51.961 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:51 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4191329433' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-26T04:49:51.961 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:51 smithi038 ceph-mon[96929]: pgmap v117: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:51.962 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:51 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4191329433' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-26T04:49:52.769 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'sleep 60' 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4191329433' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: pgmap v119: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: pgmap v120: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[93172]: Reconfiguring daemon mon.a on smithi038 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4191329433' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-26T04:49:52.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: pgmap v119: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: pgmap v120: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:52.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:52 smithi038 ceph-mon[96929]: Reconfiguring daemon mon.a on smithi038 2024-04-26T04:49:53.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4191329433' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-26T04:49:53.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: osdmap e84: 8 total, 8 up, 8 in 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='client.44153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: pgmap v119: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: pgmap v120: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: Reconfiguring mon.a (monmap changed)... 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:53.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:53.108 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:52 smithi055 ceph-mon[86906]: Reconfiguring daemon mon.a on smithi038 2024-04-26T04:49:53.945 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-04-26T04:49:53.974 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:54.066 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:53 smithi038 ceph-mon[93172]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:49:54.067 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:53 smithi038 ceph-mon[93172]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:49:54.067 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:53 smithi038 ceph-mon[93172]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:49:54.067 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:53 smithi038 ceph-mon[96929]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:49:54.067 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:53 smithi038 ceph-mon[96929]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:49:54.067 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:53 smithi038 ceph-mon[96929]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:49:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:53 smithi055 ceph-mon[86906]: osdmap e85: 8 total, 8 up, 8 in 2024-04-26T04:49:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:53 smithi055 ceph-mon[86906]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:49:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:53 smithi055 ceph-mon[86906]: osdmap e86: 8 total, 8 up, 8 in 2024-04-26T04:49:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[93172]: pgmap v123: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[96929]: pgmap v123: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:54.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:55.086 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:49:55.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:54 smithi055 ceph-mon[86906]: pgmap v123: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:55.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:55.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:55.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:49:55.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:49:55.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:56.038 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:55 smithi038 ceph-mon[93172]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:49:56.038 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:55 smithi038 ceph-mon[93172]: Reconfiguring daemon mgr.y on smithi038 2024-04-26T04:49:56.038 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:55 smithi038 ceph-mon[96929]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:49:56.038 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:55 smithi038 ceph-mon[96929]: Reconfiguring daemon mgr.y on smithi038 2024-04-26T04:49:56.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:55 smithi055 ceph-mon[86906]: Reconfiguring mgr.y (monmap changed)... 2024-04-26T04:49:56.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:55 smithi055 ceph-mon[86906]: Reconfiguring daemon mgr.y on smithi038 2024-04-26T04:49:57.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:56 smithi055 ceph-mon[86906]: pgmap v124: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:57.123 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:56 smithi038 ceph-mon[93172]: pgmap v124: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:57.123 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:56 smithi038 ceph-mon[96929]: pgmap v124: 89 pgs: 5 creating+peering, 84 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:57.124 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:49:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:49:56] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.43.0" 2024-04-26T04:49:59.149 INFO:teuthology.orchestra.run.smithi038.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: pgmap v125: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: Reconfiguring daemon mon.c on smithi038 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3186510238' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[93172]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: pgmap v125: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: Reconfiguring daemon mon.c on smithi038 2024-04-26T04:49:59.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3186510238' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:49:59.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:49:59 smithi038 ceph-mon[96929]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:49:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: pgmap v125: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: Reconfiguring mon.c (monmap changed)... 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: Reconfiguring daemon mon.c on smithi038 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3186510238' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:49:59.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:49:59 smithi055 ceph-mon[86906]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: application not enabled on pool 'unique_pool_2' 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[96929]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: application not enabled on pool 'unique_pool_2' 2024-04-26T04:50:00.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:00 smithi038 ceph-mon[93172]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:50:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: osdmap e87: 8 total, 8 up, 8 in 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: daemon grafana.a on smithi055 is in error state 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: application not enabled on pool 'unique_pool_2' 2024-04-26T04:50:00.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:00 smithi055 ceph-mon[86906]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-04-26T04:50:00.901 DEBUG:teuthology.orchestra.run.smithi038:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-04-26T04:50:00.903 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-26T04:50:00.904 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-26T04:50:00.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:hints = 1 2024-04-26T04:50:00.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-26T04:50:00.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object prefix: benchmark_data_smithi038_103292 2024-04-26T04:50:00.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:50:00.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 0 0 0 0 0 0 - 0 2024-04-26T04:50:01.332 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[93172]: pgmap v127: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:01.332 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[93172]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[96929]: pgmap v127: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[96929]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:50:01.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:01 smithi055 ceph-mon[86906]: pgmap v127: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:01 smithi055 ceph-mon[86906]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:01.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:50:01.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:01.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 1 16 3713 3697 231.049 231.062 0.00290011 0.00431088 2024-04-26T04:50:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:02 smithi038 ceph-mon[96929]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:50:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:02 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.0 on smithi038 2024-04-26T04:50:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:02 smithi038 ceph-mon[93172]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:50:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:02 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.0 on smithi038 2024-04-26T04:50:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:02 smithi055 ceph-mon[86906]: Reconfiguring osd.0 (monmap changed)... 2024-04-26T04:50:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:02 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.0 on smithi038 2024-04-26T04:50:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:02.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 2 16 6787 6771 211.578 192.125 0.00530979 0.00471438 2024-04-26T04:50:03.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[93172]: pgmap v128: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:03.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:03.209 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[96929]: pgmap v128: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:03.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:03 smithi055 ceph-mon[86906]: pgmap v128: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:50:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:03.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:50:03.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:03.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 3 16 9815 9799 204.13 189.25 0.00465102 0.00489045 2024-04-26T04:50:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:04 smithi038 ceph-mon[93172]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:50:04.476 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:04 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.1 on smithi038 2024-04-26T04:50:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:04 smithi038 ceph-mon[96929]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:50:04.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:04 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.1 on smithi038 2024-04-26T04:50:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:04 smithi055 ceph-mon[86906]: Reconfiguring osd.1 (monmap changed)... 2024-04-26T04:50:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:04 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.1 on smithi038 2024-04-26T04:50:04.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 4 16 12945 12929 201.999 195.625 0.00346438 0.00494452 2024-04-26T04:50:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:05 smithi038 ceph-mon[93172]: pgmap v129: 89 pgs: 89 active+clean; 423 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2024-04-26T04:50:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:05 smithi038 ceph-mon[96929]: pgmap v129: 89 pgs: 89 active+clean; 423 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2024-04-26T04:50:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:05 smithi055 ceph-mon[86906]: pgmap v129: 89 pgs: 89 active+clean; 423 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2024-04-26T04:50:05.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 5 16 15564 15548 194.334 163.688 0.00295376 0.00513816 2024-04-26T04:50:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:06 smithi055 ceph-mon[86906]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:50:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:50:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:06 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.2 on smithi038 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[93172]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.2 on smithi038 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:06.616 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:06.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[96929]: Reconfiguring osd.2 (monmap changed)... 2024-04-26T04:50:06.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:50:06.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:06.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.2 on smithi038 2024-04-26T04:50:06.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 6 16 18225 18209 189.661 166.312 0.00479271 0.00526524 2024-04-26T04:50:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:50:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:50:06] "GET /metrics HTTP/1.1" 200 36054 "" "Prometheus/2.43.0" 2024-04-26T04:50:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:07 smithi055 ceph-mon[86906]: pgmap v130: 89 pgs: 89 active+clean; 423 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2024-04-26T04:50:07.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:07 smithi038 ceph-mon[93172]: pgmap v130: 89 pgs: 89 active+clean; 423 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2024-04-26T04:50:07.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:07 smithi038 ceph-mon[96929]: pgmap v130: 89 pgs: 89 active+clean; 423 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2024-04-26T04:50:07.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 7 15 20894 20879 186.403 166.875 0.0150522 0.00535682 2024-04-26T04:50:08.959 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 8 16 23225 23209 181.304 145.625 0.00302545 0.0055091 2024-04-26T04:50:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:09 smithi055 ceph-mon[86906]: pgmap v131: 89 pgs: 89 active+clean; 759 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:50:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[93172]: pgmap v131: 89 pgs: 89 active+clean; 759 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[96929]: pgmap v131: 89 pgs: 89 active+clean; 759 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:09.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:09.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:50:09.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:09.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 9 16 25029 25013 173.686 112.75 0.00345555 0.00575155 2024-04-26T04:50:10.577 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:10 smithi038 ceph-mon[93172]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:50:10.577 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:10 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.3 on smithi038 2024-04-26T04:50:10.578 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:10 smithi038 ceph-mon[96929]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:50:10.578 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:10 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.3 on smithi038 2024-04-26T04:50:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:10 smithi055 ceph-mon[86906]: Reconfiguring osd.3 (monmap changed)... 2024-04-26T04:50:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:10 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.3 on smithi038 2024-04-26T04:50:10.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 10 16 26902 26886 168.023 117.062 0.00807272 0.00594539 2024-04-26T04:50:11.582 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:11 smithi038 ceph-mon[93172]: pgmap v132: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-04-26T04:50:11.582 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:11 smithi038 ceph-mon[96929]: pgmap v132: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-04-26T04:50:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:11 smithi055 ceph-mon[86906]: pgmap v132: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2024-04-26T04:50:11.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 11 16 28749 28733 163.241 115.438 0.00628544 0.00611967 2024-04-26T04:50:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:12 smithi055 ceph-mon[86906]: pgmap v133: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-04-26T04:50:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:12 smithi038 ceph-mon[93172]: pgmap v133: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-04-26T04:50:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:12 smithi038 ceph-mon[96929]: pgmap v133: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 108 MiB/s wr, 1.74k op/s 2024-04-26T04:50:12.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 12 15 30792 30777 160.283 127.75 0.00532656 0.00623373 2024-04-26T04:50:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:13 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:13 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:13 smithi055 ceph-mon[86906]: Reconfiguring mds.foo.smithi038.qsffiu (monmap changed)... 2024-04-26T04:50:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:13 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:50:13.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:13 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:13.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:13 smithi055 ceph-mon[86906]: Reconfiguring daemon mds.foo.smithi038.qsffiu on smithi038 2024-04-26T04:50:13.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 13 16 32544 32528 156.371 109.438 0.0129372 0.00638908 2024-04-26T04:50:13.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:13.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:13.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[93172]: Reconfiguring mds.foo.smithi038.qsffiu (monmap changed)... 2024-04-26T04:50:13.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[93172]: Reconfiguring daemon mds.foo.smithi038.qsffiu on smithi038 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[96929]: Reconfiguring mds.foo.smithi038.qsffiu (monmap changed)... 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi038.qsffiu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:13.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:13 smithi038 ceph-mon[96929]: Reconfiguring daemon mds.foo.smithi038.qsffiu on smithi038 2024-04-26T04:50:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:14 smithi055 ceph-mon[86906]: pgmap v134: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-04-26T04:50:14.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 14 16 34445 34429 153.688 118.812 0.00783355 0.00650068 2024-04-26T04:50:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:14 smithi038 ceph-mon[93172]: pgmap v134: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-04-26T04:50:14.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:14 smithi038 ceph-mon[96929]: pgmap v134: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-04-26T04:50:15.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 15 16 36325 36309 151.274 117.5 0.0114148 0.00660542 2024-04-26T04:50:16.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 16 16 38248 38232 149.331 120.188 0.00636247 0.00669141 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:50:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:50:16] "GET /metrics HTTP/1.1" 200 36113 "" "Prometheus/2.43.0" 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: pgmap v135: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: pgmap v135: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: pgmap v135: 89 pgs: 89 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-04-26T04:50:17.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:17.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:17.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-26T04:50:17.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-26T04:50:17.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:17.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:17.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 17 16 40113 40097 147.402 116.562 0.00229102 0.00676177 2024-04-26T04:50:18.286 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:18 smithi055 ceph-mon[86906]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:50:18.286 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:18 smithi055 ceph-mon[86906]: Reconfiguring daemon mon.b on smithi055 2024-04-26T04:50:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:18 smithi038 ceph-mon[93172]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:50:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:18 smithi038 ceph-mon[93172]: Reconfiguring daemon mon.b on smithi055 2024-04-26T04:50:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:18 smithi038 ceph-mon[96929]: Reconfiguring mon.b (monmap changed)... 2024-04-26T04:50:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:18 smithi038 ceph-mon[96929]: Reconfiguring daemon mon.b on smithi055 2024-04-26T04:50:18.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 18 16 41995 41979 145.747 117.625 0.00696476 0.00685579 2024-04-26T04:50:19.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:19 smithi055 ceph-mon[86906]: pgmap v136: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-04-26T04:50:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:19 smithi038 ceph-mon[96929]: pgmap v136: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-04-26T04:50:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:19 smithi038 ceph-mon[93172]: pgmap v136: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2024-04-26T04:50:19.960 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 19 16 43789 43773 143.977 112.125 0.0099602 0.00694048 2024-04-26T04:50:20.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:50:20.960861+0000 min lat: 0.00121495 max lat: 0.131138 avg lat: 0.00701522 2024-04-26T04:50:20.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:50:20.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 20 16 45603 45587 142.446 113.375 0.00619021 0.00701522 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: pgmap v137: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-04-26T04:50:21.083 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:20 smithi055 ceph-mon[86906]: Reconfiguring daemon mgr.x on smithi055 2024-04-26T04:50:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:50:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:21.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: pgmap v137: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[96929]: Reconfiguring daemon mgr.x on smithi055 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: Reconfiguring mgr.x (monmap changed)... 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:21.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: pgmap v137: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2024-04-26T04:50:21.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:20 smithi038 ceph-mon[93172]: Reconfiguring daemon mgr.x on smithi055 2024-04-26T04:50:21.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 21 16 47204 47188 140.428 100.062 0.00513268 0.00711639 2024-04-26T04:50:22.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 22 16 48699 48683 138.291 93.4375 0.00952336 0.00722493 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: pgmap v138: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:23.143 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:22 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.4 on smithi055 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: pgmap v138: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:23.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.4 on smithi055 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: pgmap v138: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: Reconfiguring osd.4 (monmap changed)... 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:23.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:22 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.4 on smithi055 2024-04-26T04:50:23.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 23 16 50348 50332 136.759 103.062 0.00561835 0.00730688 2024-04-26T04:50:24.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 24 16 51997 51981 135.355 103.062 0.00779992 0.0073824 2024-04-26T04:50:25.310 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: pgmap v139: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-04-26T04:50:25.310 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:25.310 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:25.310 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:50:25.311 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:50:25.311 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:25.311 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:25 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.5 on smithi055 2024-04-26T04:50:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: pgmap v139: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-04-26T04:50:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:50:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:50:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.5 on smithi055 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: pgmap v139: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: Reconfiguring osd.5 (monmap changed)... 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:25.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:25 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.5 on smithi055 2024-04-26T04:50:25.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 25 16 53659 53643 134.095 103.875 0.00408899 0.0074517 2024-04-26T04:50:26.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 26 16 55208 55192 132.661 96.8125 0.00717387 0.0075322 2024-04-26T04:50:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:50:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:50:26] "GET /metrics HTTP/1.1" 200 36113 "" "Prometheus/2.43.0" 2024-04-26T04:50:27.452 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: pgmap v140: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:50:27.452 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:27.453 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:27.453 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:50:27.453 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:50:27.453 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:27.453 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:27 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.6 on smithi055 2024-04-26T04:50:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: pgmap v140: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:50:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.6 on smithi055 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: pgmap v140: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: Reconfiguring osd.6 (monmap changed)... 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:27.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:27 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.6 on smithi055 2024-04-26T04:50:27.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 27 16 56772 56756 131.368 97.75 0.00683404 0.00760651 2024-04-26T04:50:28.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 28 16 58258 58242 129.993 92.875 0.00962811 0.0076872 2024-04-26T04:50:29.499 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:29 smithi055 ceph-mon[86906]: pgmap v141: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-04-26T04:50:29.499 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:29.499 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:29.499 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:50:29.499 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[93172]: pgmap v141: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[96929]: pgmap v141: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:29.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-26T04:50:29.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:29.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 29 16 59780 59764 128.79 95.125 0.0162309 0.00775968 2024-04-26T04:50:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:30 smithi055 ceph-mon[86906]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:50:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:30 smithi055 ceph-mon[86906]: Reconfiguring daemon osd.7 on smithi055 2024-04-26T04:50:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:30 smithi038 ceph-mon[93172]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:50:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:30 smithi038 ceph-mon[93172]: Reconfiguring daemon osd.7 on smithi055 2024-04-26T04:50:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:30 smithi038 ceph-mon[96929]: Reconfiguring osd.7 (monmap changed)... 2024-04-26T04:50:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:30 smithi038 ceph-mon[96929]: Reconfiguring daemon osd.7 on smithi055 2024-04-26T04:50:30.961 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 30 16 61280 61264 127.622 93.75 0.0089863 0.0078301 2024-04-26T04:50:31.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:31 smithi055 ceph-mon[86906]: pgmap v142: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-04-26T04:50:31.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:31 smithi038 ceph-mon[93172]: pgmap v142: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-04-26T04:50:31.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:31 smithi038 ceph-mon[96929]: pgmap v142: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-04-26T04:50:31.962 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 31 16 62721 62705 126.41 90.0625 0.00790856 0.00790453 2024-04-26T04:50:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:32.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:32.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:50:32.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:50:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:32.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:32.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:32.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.cuaegn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-26T04:50:32.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:32.962 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 32 16 64156 64140 125.262 89.6875 0.00646309 0.0079768 2024-04-26T04:50:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:33 smithi055 ceph-mon[86906]: Reconfiguring mds.foo.smithi055.cuaegn (monmap changed)... 2024-04-26T04:50:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:33 smithi055 ceph-mon[86906]: Reconfiguring daemon mds.foo.smithi055.cuaegn on smithi055 2024-04-26T04:50:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:33 smithi055 ceph-mon[86906]: pgmap v143: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:50:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:33 smithi038 ceph-mon[96929]: Reconfiguring mds.foo.smithi055.cuaegn (monmap changed)... 2024-04-26T04:50:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:33 smithi038 ceph-mon[96929]: Reconfiguring daemon mds.foo.smithi055.cuaegn on smithi055 2024-04-26T04:50:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:33 smithi038 ceph-mon[96929]: pgmap v143: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:50:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:33 smithi038 ceph-mon[93172]: Reconfiguring mds.foo.smithi055.cuaegn (monmap changed)... 2024-04-26T04:50:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:33 smithi038 ceph-mon[93172]: Reconfiguring daemon mds.foo.smithi055.cuaegn on smithi055 2024-04-26T04:50:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:33 smithi038 ceph-mon[93172]: pgmap v143: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-26T04:50:33.962 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 33 16 65553 65537 124.112 87.3125 0.0101272 0.00805083 2024-04-26T04:50:34.962 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 34 16 66844 66828 122.834 80.6875 0.00638269 0.00813613 2024-04-26T04:50:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:35 smithi055 ceph-mon[86906]: pgmap v144: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-04-26T04:50:35.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:35.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:35.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[96929]: pgmap v144: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-04-26T04:50:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:35.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[93172]: pgmap v144: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-04-26T04:50:35.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:35.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:35.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:50:35.962 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 35 15 68112 68097 121.591 79.3125 0.00779282 0.0082189 2024-04-26T04:50:36.963 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 36 16 69373 69357 120.396 78.75 0.00949547 0.00830083 2024-04-26T04:50:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:50:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:50:36] "GET /metrics HTTP/1.1" 200 36118 "" "Prometheus/2.43.0" 2024-04-26T04:50:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:37 smithi055 ceph-mon[86906]: pgmap v145: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-04-26T04:50:37.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:37 smithi038 ceph-mon[93172]: pgmap v145: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-04-26T04:50:37.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:37 smithi038 ceph-mon[96929]: pgmap v145: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-04-26T04:50:37.963 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 37 16 70597 70581 119.209 76.5 0.0112384 0.00838229 2024-04-26T04:50:38.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 38 16 71871 71855 118.167 79.625 0.00768385 0.00845584 2024-04-26T04:50:39.543 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:39 smithi038 ceph-mon[93172]: pgmap v146: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:50:39.544 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:39 smithi038 ceph-mon[96929]: pgmap v146: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:50:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:39 smithi055 ceph-mon[86906]: pgmap v146: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-04-26T04:50:39.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 39 16 73128 73112 117.152 78.5625 0.0108933 0.00853085 2024-04-26T04:50:40.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:50:40.964091+0000 min lat: 0.00121495 max lat: 0.131138 avg lat: 0.00860577 2024-04-26T04:50:40.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:50:40.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 40 16 74342 74326 116.12 75.875 0.0101352 0.00860577 2024-04-26T04:50:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:41 smithi055 ceph-mon[86906]: pgmap v147: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-26T04:50:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:41 smithi038 ceph-mon[93172]: pgmap v147: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-26T04:50:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:41 smithi038 ceph-mon[96929]: pgmap v147: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-26T04:50:41.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:41.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:41.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 41 16 75468 75452 115.004 70.375 0.00804154 0.00869011 2024-04-26T04:50:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:50:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:50:42.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:50:42.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 42 16 76709 76693 114.112 77.5625 0.00605017 0.00875779 2024-04-26T04:50:43.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:43 smithi055 ceph-mon[86906]: pgmap v148: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-26T04:50:43.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:43 smithi055 ceph-mon[86906]: pgmap v149: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-26T04:50:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:43 smithi038 ceph-mon[93172]: pgmap v148: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-26T04:50:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:43 smithi038 ceph-mon[93172]: pgmap v149: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-26T04:50:43.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:43 smithi038 ceph-mon[96929]: pgmap v148: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-04-26T04:50:43.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:43 smithi038 ceph-mon[96929]: pgmap v149: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-26T04:50:43.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 43 16 77975 77959 113.298 79.125 0.0187679 0.00882072 2024-04-26T04:50:44.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 44 16 79219 79203 112.49 77.75 0.00769463 0.00888474 2024-04-26T04:50:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:45 smithi055 ceph-mon[86906]: pgmap v150: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:50:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:45 smithi038 ceph-mon[93172]: pgmap v150: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:50:45.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:45 smithi038 ceph-mon[96929]: pgmap v150: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:50:45.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 45 16 80439 80423 111.685 76.25 0.0132623 0.00894792 2024-04-26T04:50:46.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 46 16 81651 81635 110.904 75.75 0.0179125 0.00901232 2024-04-26T04:50:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:50:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:50:46] "GET /metrics HTTP/1.1" 200 36118 "" "Prometheus/2.43.0" 2024-04-26T04:50:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:47 smithi055 ceph-mon[86906]: pgmap v151: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:50:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:47 smithi038 ceph-mon[93172]: pgmap v151: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:50:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:47.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:47 smithi038 ceph-mon[96929]: pgmap v151: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:50:47.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:50:47.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 47 16 82850 82834 110.138 74.9375 0.0096414 0.00907351 2024-04-26T04:50:48.964 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 48 16 84037 84021 109.389 74.1875 0.0119525 0.00913611 2024-04-26T04:50:49.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:49 smithi055 ceph-mon[86906]: pgmap v152: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-26T04:50:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:49 smithi038 ceph-mon[93172]: pgmap v152: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-26T04:50:49.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:49 smithi038 ceph-mon[96929]: pgmap v152: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-26T04:50:49.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 49 16 85225 85209 108.672 74.25 0.0178369 0.00919688 2024-04-26T04:50:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:50 smithi038 ceph-mon[93172]: pgmap v153: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:50.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:50 smithi038 ceph-mon[96929]: pgmap v153: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:50 smithi055 ceph-mon[86906]: pgmap v153: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:50.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 50 16 86384 86368 107.947 72.4375 0.0051218 0.00925808 2024-04-26T04:50:51.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 51 16 87548 87532 107.257 72.75 0.00630026 0.009318 2024-04-26T04:50:52.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 52 16 88685 88669 106.561 71.0625 0.0103521 0.00937899 2024-04-26T04:50:53.426 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:53 smithi038 ceph-mon[93172]: pgmap v154: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:53.426 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:53 smithi038 ceph-mon[96929]: pgmap v154: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:53 smithi055 ceph-mon[86906]: pgmap v154: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-26T04:50:53.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 53 16 89733 89717 105.786 65.5 0.0298664 0.00944732 2024-04-26T04:50:54.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 54 16 90694 90678 104.939 60.0625 0.0254538 0.00952372 2024-04-26T04:50:55.425 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:55 smithi038 ceph-mon[93172]: pgmap v155: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-26T04:50:55.425 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:55 smithi038 ceph-mon[96929]: pgmap v155: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-26T04:50:55.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:55 smithi055 ceph-mon[86906]: pgmap v155: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-04-26T04:50:55.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 55 16 91742 91726 104.222 65.5 0.00688556 0.00958947 2024-04-26T04:50:56.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 56 16 92856 92840 103.604 69.625 0.0122026 0.00964702 2024-04-26T04:50:57.181 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:50:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:50:56] "GET /metrics HTTP/1.1" 200 36118 "" "Prometheus/2.43.0" 2024-04-26T04:50:57.181 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:57 smithi038 ceph-mon[93172]: pgmap v156: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-04-26T04:50:57.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:57 smithi038 ceph-mon[96929]: pgmap v156: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-04-26T04:50:57.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:57 smithi055 ceph-mon[86906]: pgmap v156: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1k op/s 2024-04-26T04:50:57.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 57 16 93937 93921 102.972 67.5625 0.0250369 0.00970666 2024-04-26T04:50:58.965 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 58 16 94948 94932 102.286 63.1875 0.00964212 0.00977099 2024-04-26T04:50:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:50:59 smithi038 ceph-mon[93172]: pgmap v157: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-26T04:50:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:50:59 smithi038 ceph-mon[96929]: pgmap v157: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-26T04:50:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:50:59 smithi055 ceph-mon[86906]: pgmap v157: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-04-26T04:50:59.611 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade resume' 2024-04-26T04:50:59.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 59 16 95954 95938 101.611 62.875 0.0069488 0.0098351 2024-04-26T04:51:00.134 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:00.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:51:00.969512+0000 min lat: 0.00121495 max lat: 0.131138 avg lat: 0.0099039 2024-04-26T04:51:00.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:51:00.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 60 16 96908 96892 100.912 59.625 0.0141679 0.0099039 2024-04-26T04:51:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:01 smithi038 ceph-mon[93172]: pgmap v158: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-26T04:51:01.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:01 smithi038 ceph-mon[96929]: pgmap v158: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-26T04:51:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:01 smithi055 ceph-mon[86906]: pgmap v158: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-04-26T04:51:01.970 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 61 16 97937 97921 100.312 64.3125 0.022834 0.00996327 2024-04-26T04:51:02.397 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:02.398 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:02.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 62 15 98881 98866 99.6462 59.0625 0.0274923 0.0100301 2024-04-26T04:51:03.074 INFO:teuthology.orchestra.run.smithi038.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:03 smithi038 ceph-mon[93172]: pgmap v159: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-04-26T04:51:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:03 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:03.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:03 smithi038 ceph-mon[96929]: pgmap v159: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-04-26T04:51:03.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:03.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:03 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:03 smithi055 ceph-mon[86906]: pgmap v159: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s 2024-04-26T04:51:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:03 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:03.848 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'echo wait for half of osds to upgrade' 2024-04-26T04:51:03.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 63 15 99861 99846 99.0367 61.25 0.0142119 0.0100916 2024-04-26T04:51:04.326 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:04.514 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:04 smithi038 ceph-mon[93172]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:04.514 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:04 smithi038 ceph-mon[93172]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:04.514 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:04 smithi038 ceph-mon[96929]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:04.514 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:04 smithi038 ceph-mon[96929]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:04.573 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:04 smithi055 ceph-mon[86906]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:51:04.573 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:04 smithi055 ceph-mon[86906]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:04.969 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 64 16 100803 100787 98.4081 58.8125 0.00959287 0.0101564 2024-04-26T04:51:05.518 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:05 smithi038 ceph-mon[93172]: pgmap v160: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-26T04:51:05.519 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:05 smithi038 ceph-mon[96929]: pgmap v160: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-26T04:51:05.540 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:05 smithi055 ceph-mon[86906]: pgmap v160: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-04-26T04:51:05.970 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 65 16 101775 101759 97.8287 60.75 0.0144361 0.0102166 2024-04-26T04:51:06.746 INFO:teuthology.orchestra.run.smithi038.stdout:wait for half of osds to upgrade 2024-04-26T04:51:06.970 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 66 16 102741 102725 97.2612 60.375 0.00890515 0.0102762 2024-04-26T04:51:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:51:06] "GET /metrics HTTP/1.1" 200 36119 "" "Prometheus/2.43.0" 2024-04-26T04:51:07.570 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:07 smithi038 ceph-mon[93172]: pgmap v161: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-04-26T04:51:07.581 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:07 smithi038 ceph-mon[96929]: pgmap v161: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-04-26T04:51:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:07 smithi055 ceph-mon[86906]: pgmap v161: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-04-26T04:51:07.616 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-26T04:51:07.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 67 16 103682 103666 96.6866 58.8125 0.0253164 0.0103376 2024-04-26T04:51:08.091 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:51:08.970 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 68 16 104598 104582 96.1066 57.25 0.0117325 0.0104001 2024-04-26T04:51:09.565 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:09 smithi038 ceph-mon[93172]: pgmap v162: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-26T04:51:09.565 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:09.566 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:09.566 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:09 smithi038 ceph-mon[96929]: pgmap v162: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-26T04:51:09.566 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:09.566 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:09.572 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:09 smithi055 ceph-mon[86906]: pgmap v162: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-04-26T04:51:09.572 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:09.572 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:09.970 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 69 16 105485 105469 95.5172 55.4375 0.0246356 0.0104635 2024-04-26T04:51:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:51:10.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:10.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:10.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:51:10.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 70 16 106268 106252 94.8517 48.9375 0.00857249 0.0105353 2024-04-26T04:51:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:11 smithi055 ceph-mon[86906]: pgmap v163: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-04-26T04:51:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:11 smithi055 ceph-mon[86906]: Upgrade: Setting container_image for all mon 2024-04-26T04:51:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:11 smithi055 ceph-mon[86906]: Upgrade: Setting container_image for all crash 2024-04-26T04:51:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:11 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:51:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:11 smithi055 ceph-mon[86906]: Upgrade: osd.0 is safe to restart 2024-04-26T04:51:11.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:11 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/399844016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:11.689 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[93172]: pgmap v163: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-04-26T04:51:11.689 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[93172]: Upgrade: Setting container_image for all mon 2024-04-26T04:51:11.689 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[93172]: Upgrade: Setting container_image for all crash 2024-04-26T04:51:11.689 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:51:11.689 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[93172]: Upgrade: osd.0 is safe to restart 2024-04-26T04:51:11.689 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/399844016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:11.690 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[96929]: pgmap v163: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 997 op/s 2024-04-26T04:51:11.690 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[96929]: Upgrade: Setting container_image for all mon 2024-04-26T04:51:11.690 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[96929]: Upgrade: Setting container_image for all crash 2024-04-26T04:51:11.690 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-26T04:51:11.690 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[96929]: Upgrade: osd.0 is safe to restart 2024-04-26T04:51:11.690 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:11 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/399844016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:11.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 71 16 107118 107102 94.264 53.125 0.00361629 0.0106002 2024-04-26T04:51:12.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 72 16 107962 107946 93.6874 52.75 0.0147307 0.010668 2024-04-26T04:51:13.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:13 smithi055 ceph-mon[86906]: pgmap v164: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-04-26T04:51:13.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:13 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/613373419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:13.659 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:13 smithi038 ceph-mon[93172]: pgmap v164: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-04-26T04:51:13.660 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:13 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/613373419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:13.660 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:13 smithi038 ceph-mon[96929]: pgmap v164: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2024-04-26T04:51:13.660 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:13 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/613373419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:13.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 73 16 108845 108829 93.16 55.1875 0.0161259 0.0107295 2024-04-26T04:51:14.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 74 16 109717 109701 92.6375 54.5 0.0120292 0.0107898 2024-04-26T04:51:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[93172]: pgmap v165: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:51:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[93172]: Upgrade: Updating osd.0 2024-04-26T04:51:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:51:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[93172]: Deploying daemon osd.0 on smithi038 2024-04-26T04:51:14.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[96929]: pgmap v165: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:51:14.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[96929]: Upgrade: Updating osd.0 2024-04-26T04:51:14.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:14.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:51:14.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:14.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:14 smithi038 ceph-mon[96929]: Deploying daemon osd.0 on smithi038 2024-04-26T04:51:15.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:14 smithi055 ceph-mon[86906]: pgmap v165: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-04-26T04:51:15.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:14 smithi055 ceph-mon[86906]: Upgrade: Updating osd.0 2024-04-26T04:51:15.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:14 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:15.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:14 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-26T04:51:15.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:14 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:15.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:14 smithi055 ceph-mon[86906]: Deploying daemon osd.0 on smithi038 2024-04-26T04:51:15.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 75 16 110573 110557 92.1154 53.5 0.0183064 0.0108503 2024-04-26T04:51:16.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 76 16 111426 111410 91.6048 53.3125 0.0133268 0.0109102 2024-04-26T04:51:16.980 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:51:16] "GET /metrics HTTP/1.1" 200 36118 "" "Prometheus/2.43.0" 2024-04-26T04:51:16.981 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:16 smithi038 ceph-mon[93172]: pgmap v166: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2024-04-26T04:51:16.981 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:16 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/84289691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:16.982 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:16 smithi038 ceph-mon[96929]: pgmap v166: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2024-04-26T04:51:16.982 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:16 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/84289691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:17.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:16 smithi055 ceph-mon[86906]: pgmap v166: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2024-04-26T04:51:17.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:16 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/84289691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:17.971 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 77 16 112295 112279 91.1205 54.3125 0.0235295 0.0109681 2024-04-26T04:51:17.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:17.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:18.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:18.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 78 16 113051 113035 90.558 47.25 0.0224174 0.0110369 2024-04-26T04:51:18.973 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:18 smithi038 systemd[1]: Stopping Ceph osd.0 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:51:18.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:18 smithi038 ceph-mon[93172]: pgmap v167: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2024-04-26T04:51:18.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:18 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1327255631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:18.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:18 smithi038 ceph-mon[96929]: pgmap v167: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2024-04-26T04:51:18.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:18 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1327255631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:19.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:18 smithi055 ceph-mon[86906]: pgmap v167: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2024-04-26T04:51:19.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:18 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1327255631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:19.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:19 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[40673]: 2024-04-26T04:51:19.158+0000 7f07837b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:51:19.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:19 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[40673]: 2024-04-26T04:51:19.158+0000 7f07837b4700 -1 osd.0 87 *** Got signal Terminated *** 2024-04-26T04:51:19.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:19 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[40673]: 2024-04-26T04:51:19.158+0000 7f07837b4700 -1 osd.0 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:51:19.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 79 16 113786 113770 89.9932 45.9375 0.00374856 0.0110783 2024-04-26T04:51:20.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:19 smithi055 ceph-mon[86906]: osd.0 marked itself down and dead 2024-04-26T04:51:20.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:19 smithi038 ceph-mon[93172]: osd.0 marked itself down and dead 2024-04-26T04:51:20.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:19 smithi038 ceph-mon[96929]: osd.0 marked itself down and dead 2024-04-26T04:51:20.696 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:20 smithi038 podman[105286]: 2024-04-26 04:51:20.350252313 +0000 UTC m=+1.349408092 container stop 7a0711df145cdaa3e79c16e882eee5f4fbfbdb2569c0cbedb4108072b41a9c93 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T04:51:20.697 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:20 smithi038 podman[105286]: 2024-04-26 04:51:20.375754361 +0000 UTC m=+1.374910131 container died 7a0711df145cdaa3e79c16e882eee5f4fbfbdb2569c0cbedb4108072b41a9c93 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T04:51:20.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:51:20.972097+0000 min lat: 0.00121495 max lat: 1.06723 avg lat: 0.011227 2024-04-26T04:51:20.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:51:20.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 80 16 113991 113975 89.0284 12.8125 0.0343484 0.011227 2024-04-26T04:51:20.973 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:20 smithi038 ceph-mon[93172]: pgmap v168: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:51:20.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:20 smithi038 ceph-mon[93172]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:20.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:20 smithi038 ceph-mon[93172]: osdmap e88: 8 total, 7 up, 8 in 2024-04-26T04:51:20.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:20 smithi038 ceph-mon[96929]: pgmap v168: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:51:20.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:20 smithi038 ceph-mon[96929]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:20.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:20 smithi038 ceph-mon[96929]: osdmap e88: 8 total, 7 up, 8 in 2024-04-26T04:51:20.974 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:20 smithi038 podman[105286]: 2024-04-26 04:51:20.897391065 +0000 UTC m=+1.896546839 container cleanup 7a0711df145cdaa3e79c16e882eee5f4fbfbdb2569c0cbedb4108072b41a9c93 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-26T04:51:20.974 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:20 smithi038 bash[105286]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0 2024-04-26T04:51:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:20 smithi055 ceph-mon[86906]: pgmap v168: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2024-04-26T04:51:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:20 smithi055 ceph-mon[86906]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:51:21.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:20 smithi055 ceph-mon[86906]: osdmap e88: 8 total, 7 up, 8 in 2024-04-26T04:51:21.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:21 smithi038 podman[105300]: 2024-04-26 04:51:21.114375549 +0000 UTC m=+0.758656009 container remove 7a0711df145cdaa3e79c16e882eee5f4fbfbdb2569c0cbedb4108072b41a9c93 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:21.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 81 16 115155 115139 88.8274 72.75 0.00848291 0.0112526 2024-04-26T04:51:22.035 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:21 smithi038 ceph-mon[93172]: osdmap e89: 8 total, 7 up, 8 in 2024-04-26T04:51:22.035 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:21 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1003523190' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:22.036 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:21 smithi038 ceph-mon[96929]: osdmap e89: 8 total, 7 up, 8 in 2024-04-26T04:51:22.036 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:21 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1003523190' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:22.036 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:21 smithi038 podman[105398]: 2024-04-26 04:51:21.753210189 +0000 UTC m=+0.033561525 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:22.036 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:21 smithi038 podman[105398]: 2024-04-26 04:51:21.873994345 +0000 UTC m=+0.154345670 container create 1673734be23b5f22fde8ad384ca4ceecf276c36dbe86df99bbfcf774b113d26b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:21 smithi055 ceph-mon[86906]: osdmap e89: 8 total, 7 up, 8 in 2024-04-26T04:51:22.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:21 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1003523190' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:22.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:22 smithi038 podman[105398]: 2024-04-26 04:51:22.152812498 +0000 UTC m=+0.433163827 container init 1673734be23b5f22fde8ad384ca4ceecf276c36dbe86df99bbfcf774b113d26b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T04:51:22.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:22 smithi038 podman[105398]: 2024-04-26 04:51:22.158059469 +0000 UTC m=+0.438410796 container start 1673734be23b5f22fde8ad384ca4ceecf276c36dbe86df99bbfcf774b113d26b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T04:51:22.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:22 smithi038 podman[105398]: 2024-04-26 04:51:22.261144155 +0000 UTC m=+0.541495477 container attach 1673734be23b5f22fde8ad384ca4ceecf276c36dbe86df99bbfcf774b113d26b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T04:51:22.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 82 16 116299 116283 88.616 71.5 0.00741023 0.0112795 2024-04-26T04:51:22.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:22 smithi038 ceph-mon[93172]: pgmap v171: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s 2024-04-26T04:51:22.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:22 smithi038 ceph-mon[96929]: pgmap v171: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s 2024-04-26T04:51:22.974 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:22 smithi038 podman[105421]: 2024-04-26 04:51:22.72674261 +0000 UTC m=+0.045788779 container died 1673734be23b5f22fde8ad384ca4ceecf276c36dbe86df99bbfcf774b113d26b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb) 2024-04-26T04:51:23.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:22 smithi055 ceph-mon[86906]: pgmap v171: 89 pgs: 15 stale+active+clean, 74 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s 2024-04-26T04:51:23.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:23 smithi038 podman[105421]: 2024-04-26 04:51:23.383645482 +0000 UTC m=+0.702691645 container remove 1673734be23b5f22fde8ad384ca4ceecf276c36dbe86df99bbfcf774b113d26b (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-26T04:51:23.725 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:23 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-26T04:51:23.725 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:23 smithi038 systemd[1]: Stopped Ceph osd.0 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:51:23.725 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:23 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.0.service: Consumed 15min 49.543s CPU time. 2024-04-26T04:51:23.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 83 16 117372 117356 88.3563 67.0625 0.011861 0.0113126 2024-04-26T04:51:24.095 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:23 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2897707854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:24.095 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:23 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2897707854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:24.095 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:23 smithi038 systemd[1]: Starting Ceph osd.0 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:51:24.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:23 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2897707854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:24.417 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:24 smithi038 podman[105533]: 2024-04-26 04:51:24.007209266 +0000 UTC m=+0.050528813 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:24.417 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:24 smithi038 podman[105533]: 2024-04-26 04:51:24.136011371 +0000 UTC m=+0.179330950 container create f14f760e380fb88fd5e27977c2b3dae011e94981a8a39546c1291ff5e5203eae (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:51:24.417 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:24 smithi038 podman[105533]: 2024-04-26 04:51:24.416279045 +0000 UTC m=+0.459598600 container init f14f760e380fb88fd5e27977c2b3dae011e94981a8a39546c1291ff5e5203eae (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-26T04:51:24.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:24 smithi038 podman[105533]: 2024-04-26 04:51:24.422974066 +0000 UTC m=+0.466293600 container start f14f760e380fb88fd5e27977c2b3dae011e94981a8a39546c1291ff5e5203eae (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-26T04:51:24.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:24 smithi038 podman[105533]: 2024-04-26 04:51:24.531197809 +0000 UTC m=+0.574517337 container attach f14f760e380fb88fd5e27977c2b3dae011e94981a8a39546c1291ff5e5203eae (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:51:24.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 84 16 118401 118385 88.07 64.3125 0.0152019 0.0113485 2024-04-26T04:51:25.033 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:24 smithi038 ceph-mon[96929]: pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:25.034 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:24 smithi038 ceph-mon[96929]: Health check failed: Degraded data redundancy: 58299/349083 objects degraded (16.701%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:25.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:24 smithi038 ceph-mon[93172]: pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:25.034 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:24 smithi038 ceph-mon[93172]: Health check failed: Degraded data redundancy: 58299/349083 objects degraded (16.701%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:24 smithi055 ceph-mon[86906]: pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:25.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:24 smithi055 ceph-mon[86906]: Health check failed: Degraded data redundancy: 58299/349083 objects degraded (16.701%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 bash[105533]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T04:51:25.311 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-26T04:51:25.312 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-26T04:51:25.312 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-26T04:51:25.312 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-26T04:51:25.312 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate[105547]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-26T04:51:25.707 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:25 smithi038 podman[105533]: 2024-04-26 04:51:25.310125286 +0000 UTC m=+1.353444821 container died f14f760e380fb88fd5e27977c2b3dae011e94981a8a39546c1291ff5e5203eae (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True) 2024-04-26T04:51:25.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 85 16 119327 119311 87.7147 57.875 0.0185668 0.0113949 2024-04-26T04:51:26.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 podman[105673]: 2024-04-26 04:51:26.138018712 +0000 UTC m=+0.816108527 container remove f14f760e380fb88fd5e27977c2b3dae011e94981a8a39546c1291ff5e5203eae (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-26T04:51:26.801 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 podman[105750]: 2024-04-26 04:51:26.402773006 +0000 UTC m=+0.042000146 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:51:26.802 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 podman[105750]: 2024-04-26 04:51:26.53927973 +0000 UTC m=+0.178506851 container create e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-26T04:51:26.972 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 86 16 120387 120371 87.4651 66.25 0.0104412 0.0114281 2024-04-26T04:51:27.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:26 smithi055 ceph-mon[86906]: pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:27.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1545892321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:51:26] "GET /metrics HTTP/1.1" 200 36118 "" "Prometheus/2.43.0" 2024-04-26T04:51:27.224 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 podman[105750]: 2024-04-26 04:51:26.801101467 +0000 UTC m=+0.440328603 container init e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-04-26T04:51:27.224 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 podman[105750]: 2024-04-26 04:51:26.806324725 +0000 UTC m=+0.445551850 container start e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True) 2024-04-26T04:51:27.224 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 bash[105750]: e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 2024-04-26T04:51:27.224 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:26 smithi038 systemd[1]: Started Ceph osd.0 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:51:27.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:26 smithi038 ceph-mon[93172]: pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:27.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1545892321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:26 smithi038 ceph-mon[96929]: pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1545892321' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:27.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[105760]: 2024-04-26T04:51:27.372+0000 7f9af7be57c0 -1 Falling back to public interface 2024-04-26T04:51:27.973 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 87 16 121399 121383 87.1858 63.25 0.00612007 0.0114642 2024-04-26T04:51:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.362 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:28.362 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:28.363 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.363 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:28.363 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:28.363 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:28.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 88 16 122417 122401 86.9179 63.625 0.00672758 0.0114997 2024-04-26T04:51:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:29 smithi055 ceph-mon[86906]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 31 MiB/s wr, 501 op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:29 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4242514154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:29 smithi038 ceph-mon[93172]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 31 MiB/s wr, 501 op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:29 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4242514154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:29 smithi038 ceph-mon[96929]: pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 31 MiB/s wr, 501 op/s; 58299/349083 objects degraded (16.701%) 2024-04-26T04:51:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:29 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4242514154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:29.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 89 16 123509 123493 86.7081 68.25 0.0127796 0.0115278 2024-04-26T04:51:30.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:30 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 60855/364386 objects degraded (16.701%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:30.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:30 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 60855/364386 objects degraded (16.701%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:30.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:30 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 60855/364386 objects degraded (16.701%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total time run: 90.0093 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total writes made: 124464 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Write size: 65536 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object size: 65536 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Bandwidth (MB/sec): 86.4244 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Bandwidth: 37.4861 2024-04-26T04:51:30.974 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max bandwidth (MB/sec): 231.062 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min bandwidth (MB/sec): 12.8125 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average IOPS: 1382 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev IOPS: 599.777 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max IOPS: 3697 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min IOPS: 205 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average Latency(s): 0.0115666 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Latency(s): 0.0140234 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max latency(s): 1.06723 2024-04-26T04:51:30.975 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min latency(s): 0.00121495 2024-04-26T04:51:30.980 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Cleaning up (deleting benchmark objects) 2024-04-26T04:51:31.285 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:31 smithi038 ceph-mon[93172]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 60855/364386 objects degraded (16.701%) 2024-04-26T04:51:31.286 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:31 smithi038 ceph-mon[96929]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 60855/364386 objects degraded (16.701%) 2024-04-26T04:51:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:31 smithi055 ceph-mon[86906]: pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 60855/364386 objects degraded (16.701%) 2024-04-26T04:51:32.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:32 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1216044240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:32.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:32.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:32.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:32 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1216044240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:32.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:32.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:32 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1216044240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:33 smithi055 ceph-mon[86906]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 60855/364386 objects degraded (16.701%) 2024-04-26T04:51:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:33 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:33 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:33 smithi038 ceph-mon[93172]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 60855/364386 objects degraded (16.701%) 2024-04-26T04:51:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:33.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:33 smithi038 ceph-mon[96929]: pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 60855/364386 objects degraded (16.701%) 2024-04-26T04:51:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:33 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:33.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:33 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:34 smithi055 ceph-mon[86906]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.19k op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:34 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/157648663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:34 smithi038 ceph-mon[93172]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.19k op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:34 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/157648663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:34.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:34 smithi038 ceph-mon[96929]: pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 63 MiB/s wr, 1.19k op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:34.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:34 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/157648663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:35.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:35.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:35.857 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:35.858 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:35.858 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:35.858 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:36 smithi038 ceph-mon[93172]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 850 op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2726609813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:36 smithi038 ceph-mon[96929]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 850 op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2726609813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:37.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:36 smithi055 ceph-mon[86906]: pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 850 op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:37.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2726609813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:37.166 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:51:36] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.43.0" 2024-04-26T04:51:38.474 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[105760]: 2024-04-26T04:51:38.179+0000 7f9af7be57c0 -1 osd.0 87 log_to_monitors true 2024-04-26T04:51:38.930 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-mon[93172]: pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 850 op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:38.930 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-mon[93172]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:38.930 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 61424/367206 objects degraded (16.727%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:38.930 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-mon[96929]: pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 850 op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:38.930 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-mon[96929]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:38.930 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 61424/367206 objects degraded (16.727%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:38.931 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 04:51:38 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[105760]: 2024-04-26T04:51:38.697+0000 7f9ae903d700 -1 osd.0 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:51:39.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:38 smithi055 ceph-mon[86906]: pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 850 op/s; 61424/367206 objects degraded (16.727%) 2024-04-26T04:51:39.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:38 smithi055 ceph-mon[86906]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-26T04:51:39.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:38 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 61424/367206 objects degraded (16.727%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[93172]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:51:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[93172]: osdmap e90: 8 total, 7 up, 8 in 2024-04-26T04:51:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[93172]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:51:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3886753513' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:39.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[96929]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:51:39.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[96929]: osdmap e90: 8 total, 7 up, 8 in 2024-04-26T04:51:39.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[96929]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:51:39.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:39 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3886753513' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:40.007 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:39 smithi055 ceph-mon[86906]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-26T04:51:40.007 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:39 smithi055 ceph-mon[86906]: osdmap e90: 8 total, 7 up, 8 in 2024-04-26T04:51:40.007 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:39 smithi055 ceph-mon[86906]: from='osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:51:40.007 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:39 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3886753513' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:40.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:51:40.736+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:51:40.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 970 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:40.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:40.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320] boot 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 970 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320] boot 2024-04-26T04:51:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T04:51:40.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:40.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:40.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:40.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:40.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:40.976 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 970 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:41.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: osd.0 [v2:172.21.15.38:6802/3874109320,v1:172.21.15.38:6803/3874109320] boot 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: osdmap e91: 8 total, 8 up, 8 in 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:51:41.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:51:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/686926938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:41.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:41.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:51:41.976 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/686926938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: osdmap e92: 8 total, 8 up, 8 in 2024-04-26T04:51:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:51:42.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/686926938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:42.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:42 smithi038 ceph-mon[93172]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:51:42.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:42 smithi038 ceph-mon[96929]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:51:43.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:42 smithi055 ceph-mon[86906]: osdmap e93: 8 total, 8 up, 8 in 2024-04-26T04:51:43.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:43 smithi038 ceph-mon[93172]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:43.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:43 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 59151/353424 objects degraded (16.737%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:43.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:43 smithi038 ceph-mon[96929]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:43.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:43 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 59151/353424 objects degraded (16.737%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:43 smithi055 ceph-mon[86906]: pgmap v187: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 7.2 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 59151/353424 objects degraded (16.737%) 2024-04-26T04:51:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:43 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 59151/353424 objects degraded (16.737%), 15 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3848132672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:45.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3848132672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:45.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3848132672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:46.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:45 smithi055 ceph-mon[86906]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 8958/345606 objects degraded (2.592%); 808 KiB/s, 13 objects/s recovering 2024-04-26T04:51:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:45 smithi038 ceph-mon[93172]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 8958/345606 objects degraded (2.592%); 808 KiB/s, 13 objects/s recovering 2024-04-26T04:51:46.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:45 smithi038 ceph-mon[96929]: pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 8958/345606 objects degraded (2.592%); 808 KiB/s, 13 objects/s recovering 2024-04-26T04:51:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:51:46] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-04-26T04:51:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:47 smithi055 ceph-mon[86906]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 8958/345606 objects degraded (2.592%); 661 KiB/s, 10 objects/s recovering 2024-04-26T04:51:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:47 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4271395217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:48.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[96929]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 8958/345606 objects degraded (2.592%); 661 KiB/s, 10 objects/s recovering 2024-04-26T04:51:48.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:48.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:48.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4271395217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:48.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[93172]: pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 8958/345606 objects degraded (2.592%); 661 KiB/s, 10 objects/s recovering 2024-04-26T04:51:48.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:51:48.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:51:48.227 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:47 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4271395217' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:48 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 8958/345606 objects degraded (2.592%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:48 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 8958/345606 objects degraded (2.592%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:49.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:48 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 8958/345606 objects degraded (2.592%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:49.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:49 smithi038 ceph-mon[93172]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 8706/333792 objects degraded (2.608%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T04:51:49.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:49 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/725587065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:49.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:49 smithi038 ceph-mon[96929]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 8706/333792 objects degraded (2.608%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T04:51:49.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:49 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/725587065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:49 smithi055 ceph-mon[86906]: pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 8706/333792 objects degraded (2.608%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T04:51:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:49 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/725587065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:52.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:51 smithi055 ceph-mon[86906]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8706/333792 objects degraded (2.608%); 2.0 MiB/s, 32 objects/s recovering 2024-04-26T04:51:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:51 smithi038 ceph-mon[96929]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8706/333792 objects degraded (2.608%); 2.0 MiB/s, 32 objects/s recovering 2024-04-26T04:51:52.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:51 smithi038 ceph-mon[93172]: pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 8706/333792 objects degraded (2.608%); 2.0 MiB/s, 32 objects/s recovering 2024-04-26T04:51:53.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:52 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2856986818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:53.125 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:52 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2856986818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:53.125 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:52 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2856986818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:53 smithi055 ceph-mon[86906]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 8706/333792 objects degraded (2.608%); 1.8 MiB/s, 29 objects/s recovering 2024-04-26T04:51:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:53 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 8706/333792 objects degraded (2.608%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:54.147 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:53 smithi038 ceph-mon[93172]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 8706/333792 objects degraded (2.608%); 1.8 MiB/s, 29 objects/s recovering 2024-04-26T04:51:54.147 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:53 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 8706/333792 objects degraded (2.608%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:54.147 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:53 smithi038 ceph-mon[96929]: pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 8706/333792 objects degraded (2.608%); 1.8 MiB/s, 29 objects/s recovering 2024-04-26T04:51:54.148 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:53 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 8706/333792 objects degraded (2.608%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:54 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2028099197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:55.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:54 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2028099197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:54 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2028099197' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:56.140 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:51:55.752+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:51:56.141 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:55 smithi038 ceph-mon[93172]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 8451/322185 objects degraded (2.623%); 3.0 MiB/s, 47 objects/s recovering 2024-04-26T04:51:56.141 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:55 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:56.141 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:55 smithi038 ceph-mon[96929]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 8451/322185 objects degraded (2.623%); 3.0 MiB/s, 47 objects/s recovering 2024-04-26T04:51:56.141 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:55 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:55 smithi055 ceph-mon[86906]: pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s; 8451/322185 objects degraded (2.623%); 3.0 MiB/s, 47 objects/s recovering 2024-04-26T04:51:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:55 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:56 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:56 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:51:57.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:51:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:51:56] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-04-26T04:51:57.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:56 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:57.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:56 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:51:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:56 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:51:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:56 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:51:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:57 smithi038 ceph-mon[93172]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 8451/322185 objects degraded (2.623%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:51:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:57 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1618708413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:57 smithi038 ceph-mon[96929]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 8451/322185 objects degraded (2.623%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:51:58.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:57 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1618708413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:58.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:57 smithi055 ceph-mon[86906]: pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 650 op/s; 8451/322185 objects degraded (2.623%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:51:58.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:57 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1618708413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:51:59.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:58 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 8451/322185 objects degraded (2.623%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:59.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:58 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 8451/322185 objects degraded (2.623%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:51:59.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:58 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 8451/322185 objects degraded (2.623%), 8 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:51:59 smithi038 ceph-mon[93172]: pgmap v195: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 8374/311148 objects degraded (2.691%); 3.0 MiB/s, 61 objects/s recovering 2024-04-26T04:52:00.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:51:59 smithi038 ceph-mon[96929]: pgmap v195: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 8374/311148 objects degraded (2.691%); 3.0 MiB/s, 61 objects/s recovering 2024-04-26T04:52:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:51:59 smithi055 ceph-mon[86906]: pgmap v195: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 8374/311148 objects degraded (2.691%); 3.0 MiB/s, 61 objects/s recovering 2024-04-26T04:52:01.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1730700875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:01.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1730700875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:01.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:00 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1730700875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:01 smithi038 ceph-mon[93172]: pgmap v196: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 8374/311148 objects degraded (2.691%); 1.7 MiB/s, 40 objects/s recovering 2024-04-26T04:52:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:02.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:01 smithi038 ceph-mon[96929]: pgmap v196: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 8374/311148 objects degraded (2.691%); 1.7 MiB/s, 40 objects/s recovering 2024-04-26T04:52:02.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:01 smithi055 ceph-mon[86906]: pgmap v196: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 8374/311148 objects degraded (2.691%); 1.7 MiB/s, 40 objects/s recovering 2024-04-26T04:52:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2371071917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:03.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2371071917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:03.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2371071917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:03 smithi038 ceph-mon[93172]: pgmap v197: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 8374/311148 objects degraded (2.691%); 1.7 MiB/s, 40 objects/s recovering 2024-04-26T04:52:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:03 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 8374/311148 objects degraded (2.691%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:03 smithi038 ceph-mon[96929]: pgmap v197: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 8374/311148 objects degraded (2.691%); 1.7 MiB/s, 40 objects/s recovering 2024-04-26T04:52:04.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:03 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 8374/311148 objects degraded (2.691%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:03 smithi055 ceph-mon[86906]: pgmap v197: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 8374/311148 objects degraded (2.691%); 1.7 MiB/s, 40 objects/s recovering 2024-04-26T04:52:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:03 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 8374/311148 objects degraded (2.691%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:05 smithi055 ceph-mon[86906]: pgmap v198: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 8374/299613 objects degraded (2.795%); 1.7 MiB/s, 62 objects/s recovering 2024-04-26T04:52:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:05 smithi055 ceph-mon[86906]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T04:52:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:05 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/660318326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:06.364 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:05 smithi038 ceph-mon[93172]: pgmap v198: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 8374/299613 objects degraded (2.795%); 1.7 MiB/s, 62 objects/s recovering 2024-04-26T04:52:06.364 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:05 smithi038 ceph-mon[93172]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T04:52:06.364 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:05 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/660318326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:06.365 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:05 smithi038 ceph-mon[96929]: pgmap v198: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 8374/299613 objects degraded (2.795%); 1.7 MiB/s, 62 objects/s recovering 2024-04-26T04:52:06.365 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:05 smithi038 ceph-mon[96929]: osdmap e94: 8 total, 8 up, 8 in 2024-04-26T04:52:06.365 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:05 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/660318326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:52:06] "GET /metrics HTTP/1.1" 200 36253 "" "Prometheus/2.43.0" 2024-04-26T04:52:07.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:06 smithi038 ceph-mon[93172]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T04:52:07.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:06 smithi038 ceph-mon[96929]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T04:52:07.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:06 smithi055 ceph-mon[86906]: osdmap e95: 8 total, 8 up, 8 in 2024-04-26T04:52:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:08 smithi055 ceph-mon[86906]: pgmap v201: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 8374/299613 objects degraded (2.795%); 0 B/s, 32 objects/s recovering 2024-04-26T04:52:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:08 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/137199561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:08 smithi038 ceph-mon[93172]: pgmap v201: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 8374/299613 objects degraded (2.795%); 0 B/s, 32 objects/s recovering 2024-04-26T04:52:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:08 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/137199561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:08 smithi038 ceph-mon[96929]: pgmap v201: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 8374/299613 objects degraded (2.795%); 0 B/s, 32 objects/s recovering 2024-04-26T04:52:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:08 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/137199561' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 8374/299613 objects degraded (2.795%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 8374/299613 objects degraded (2.795%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:09.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 8374/299613 objects degraded (2.795%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:10 smithi055 ceph-mon[86906]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7589/292830 objects degraded (2.592%); 2.1 MiB/s, 74 objects/s recovering 2024-04-26T04:52:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:10 smithi038 ceph-mon[93172]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7589/292830 objects degraded (2.592%); 2.1 MiB/s, 74 objects/s recovering 2024-04-26T04:52:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:10 smithi038 ceph-mon[96929]: pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 7589/292830 objects degraded (2.592%); 2.1 MiB/s, 74 objects/s recovering 2024-04-26T04:52:11.038 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:52:10.771+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:11 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3290319885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:11 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:11 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3290319885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:11 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:11.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:11 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3290319885' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:11.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:11 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:12 smithi055 ceph-mon[86906]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 7589/289656 objects degraded (2.620%); 2.1 MiB/s, 74 objects/s recovering 2024-04-26T04:52:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:12 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:12 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:12 smithi038 ceph-mon[96929]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 7589/289656 objects degraded (2.620%); 2.1 MiB/s, 74 objects/s recovering 2024-04-26T04:52:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:12 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:12 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:12.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:12 smithi038 ceph-mon[93172]: pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 7589/289656 objects degraded (2.620%); 2.1 MiB/s, 74 objects/s recovering 2024-04-26T04:52:12.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:12 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:12.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:12 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:13.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:13 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3499268551' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:13.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:13 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3499268551' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:13.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:13 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3499268551' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:14 smithi055 ceph-mon[86906]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 7589/289656 objects degraded (2.620%); 2.1 MiB/s, 42 objects/s recovering 2024-04-26T04:52:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 7589/289656 objects degraded (2.620%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:14 smithi038 ceph-mon[96929]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 7589/289656 objects degraded (2.620%); 2.1 MiB/s, 42 objects/s recovering 2024-04-26T04:52:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 7589/289656 objects degraded (2.620%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:14 smithi038 ceph-mon[93172]: pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 7589/289656 objects degraded (2.620%); 2.1 MiB/s, 42 objects/s recovering 2024-04-26T04:52:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 7589/289656 objects degraded (2.620%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:16 smithi055 ceph-mon[86906]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 7363/278328 objects degraded (2.645%); 3.2 MiB/s, 58 objects/s recovering 2024-04-26T04:52:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:16 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2777461308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:16 smithi038 ceph-mon[96929]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 7363/278328 objects degraded (2.645%); 3.2 MiB/s, 58 objects/s recovering 2024-04-26T04:52:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:16 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2777461308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:16 smithi038 ceph-mon[93172]: pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 7363/278328 objects degraded (2.645%); 3.2 MiB/s, 58 objects/s recovering 2024-04-26T04:52:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:16 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2777461308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:17.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:52:16] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-04-26T04:52:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:18 smithi055 ceph-mon[86906]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 7363/278328 objects degraded (2.645%); 2.9 MiB/s, 53 objects/s recovering 2024-04-26T04:52:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:18 smithi038 ceph-mon[93172]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 7363/278328 objects degraded (2.645%); 2.9 MiB/s, 53 objects/s recovering 2024-04-26T04:52:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:18 smithi038 ceph-mon[96929]: pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 7363/278328 objects degraded (2.645%); 2.9 MiB/s, 53 objects/s recovering 2024-04-26T04:52:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:19 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2361794904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:19 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 7363/278328 objects degraded (2.645%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:19 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2361794904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 7363/278328 objects degraded (2.645%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:19 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2361794904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:19 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 7363/278328 objects degraded (2.645%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:20.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:20 smithi038 ceph-mon[93172]: pgmap v207: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 7159/270651 objects degraded (2.645%); 3.7 MiB/s, 66 objects/s recovering 2024-04-26T04:52:20.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:20 smithi038 ceph-mon[96929]: pgmap v207: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 7159/270651 objects degraded (2.645%); 3.7 MiB/s, 66 objects/s recovering 2024-04-26T04:52:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:20 smithi055 ceph-mon[86906]: pgmap v207: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 7159/270651 objects degraded (2.645%); 3.7 MiB/s, 66 objects/s recovering 2024-04-26T04:52:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:21 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3714571273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:21.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:21 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3714571273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:21 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3714571273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:22.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:22 smithi038 ceph-mon[93172]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7159/267120 objects degraded (2.680%); 2.2 MiB/s, 38 objects/s recovering 2024-04-26T04:52:22.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:22 smithi038 ceph-mon[96929]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7159/267120 objects degraded (2.680%); 2.2 MiB/s, 38 objects/s recovering 2024-04-26T04:52:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:22 smithi055 ceph-mon[86906]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 7159/267120 objects degraded (2.680%); 2.2 MiB/s, 38 objects/s recovering 2024-04-26T04:52:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:24 smithi038 ceph-mon[93172]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7159/267120 objects degraded (2.680%); 2.2 MiB/s, 38 objects/s recovering 2024-04-26T04:52:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:24 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4011483821' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 7159/267120 objects degraded (2.680%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:24.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:24 smithi038 ceph-mon[96929]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7159/267120 objects degraded (2.680%); 2.2 MiB/s, 38 objects/s recovering 2024-04-26T04:52:24.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:24 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4011483821' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:24.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 7159/267120 objects degraded (2.680%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:24 smithi055 ceph-mon[86906]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 7159/267120 objects degraded (2.680%); 2.2 MiB/s, 38 objects/s recovering 2024-04-26T04:52:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:24 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4011483821' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:24.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 7159/267120 objects degraded (2.680%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:26.159 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:52:25.787+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:26 smithi038 ceph-mon[93172]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 7159/255849 objects degraded (2.798%); 2.2 MiB/s, 59 objects/s recovering 2024-04-26T04:52:26.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:26 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:26.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:26 smithi038 ceph-mon[96929]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 7159/255849 objects degraded (2.798%); 2.2 MiB/s, 59 objects/s recovering 2024-04-26T04:52:26.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:26 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:26 smithi055 ceph-mon[86906]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 938 op/s; 7159/255849 objects degraded (2.798%); 2.2 MiB/s, 59 objects/s recovering 2024-04-26T04:52:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:26 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:27 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:27 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:27.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:27 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2626355650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:52:26] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-04-26T04:52:27.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:27 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:27 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:27.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:27 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2626355650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:27 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:27 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:52:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:27 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2626355650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:28 smithi038 ceph-mon[93172]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 7159/255849 objects degraded (2.798%); 1.1 MiB/s, 41 objects/s recovering 2024-04-26T04:52:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:28 smithi038 ceph-mon[93172]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T04:52:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:28 smithi038 ceph-mon[96929]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 7159/255849 objects degraded (2.798%); 1.1 MiB/s, 41 objects/s recovering 2024-04-26T04:52:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:28 smithi038 ceph-mon[96929]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T04:52:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:28 smithi055 ceph-mon[86906]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 7159/255849 objects degraded (2.798%); 1.1 MiB/s, 41 objects/s recovering 2024-04-26T04:52:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:28 smithi055 ceph-mon[86906]: osdmap e96: 8 total, 8 up, 8 in 2024-04-26T04:52:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:29 smithi038 ceph-mon[93172]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:52:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:29 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 7159/255849 objects degraded (2.798%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:29 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/223098617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:29 smithi038 ceph-mon[96929]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:52:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:29 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 7159/255849 objects degraded (2.798%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:29.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:29 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/223098617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:29 smithi055 ceph-mon[86906]: osdmap e97: 8 total, 8 up, 8 in 2024-04-26T04:52:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:29 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 7159/255849 objects degraded (2.798%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:29 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/223098617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:30 smithi038 ceph-mon[93172]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6455/249393 objects degraded (2.588%); 1.4 MiB/s, 55 objects/s recovering 2024-04-26T04:52:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:30 smithi038 ceph-mon[96929]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6455/249393 objects degraded (2.588%); 1.4 MiB/s, 55 objects/s recovering 2024-04-26T04:52:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:30 smithi055 ceph-mon[86906]: pgmap v214: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6455/249393 objects degraded (2.588%); 1.4 MiB/s, 55 objects/s recovering 2024-04-26T04:52:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:32 smithi055 ceph-mon[86906]: pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 6455/245802 objects degraded (2.626%); 1.4 MiB/s, 82 objects/s recovering 2024-04-26T04:52:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:32 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1273597478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:32 smithi038 ceph-mon[93172]: pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 6455/245802 objects degraded (2.626%); 1.4 MiB/s, 82 objects/s recovering 2024-04-26T04:52:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:32 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1273597478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:32 smithi038 ceph-mon[96929]: pgmap v215: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 6455/245802 objects degraded (2.626%); 1.4 MiB/s, 82 objects/s recovering 2024-04-26T04:52:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:32 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1273597478' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:34 smithi055 ceph-mon[86906]: pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 6455/245802 objects degraded (2.626%); 1.4 MiB/s, 50 objects/s recovering 2024-04-26T04:52:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:34 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 6455/245802 objects degraded (2.626%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:34 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3964073909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:34 smithi038 ceph-mon[93172]: pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 6455/245802 objects degraded (2.626%); 1.4 MiB/s, 50 objects/s recovering 2024-04-26T04:52:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:34 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 6455/245802 objects degraded (2.626%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:34 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3964073909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:34.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:34 smithi038 ceph-mon[96929]: pgmap v216: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 6455/245802 objects degraded (2.626%); 1.4 MiB/s, 50 objects/s recovering 2024-04-26T04:52:34.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:34 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 6455/245802 objects degraded (2.626%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:34.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:34 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3964073909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:36 smithi055 ceph-mon[86906]: pgmap v217: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 6284/234936 objects degraded (2.675%); 2.7 MiB/s, 71 objects/s recovering 2024-04-26T04:52:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:36 smithi038 ceph-mon[93172]: pgmap v217: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 6284/234936 objects degraded (2.675%); 2.7 MiB/s, 71 objects/s recovering 2024-04-26T04:52:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:36 smithi038 ceph-mon[96929]: pgmap v217: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 6284/234936 objects degraded (2.675%); 2.7 MiB/s, 71 objects/s recovering 2024-04-26T04:52:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:52:36] "GET /metrics HTTP/1.1" 200 36267 "" "Prometheus/2.43.0" 2024-04-26T04:52:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:37 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2355913461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:37.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:37 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2355913461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:37.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:37 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2355913461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:38 smithi055 ceph-mon[86906]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6284/234936 objects degraded (2.675%); 2.3 MiB/s, 60 objects/s recovering 2024-04-26T04:52:38.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:38 smithi038 ceph-mon[93172]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6284/234936 objects degraded (2.675%); 2.3 MiB/s, 60 objects/s recovering 2024-04-26T04:52:38.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:38 smithi038 ceph-mon[96929]: pgmap v218: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 6284/234936 objects degraded (2.675%); 2.3 MiB/s, 60 objects/s recovering 2024-04-26T04:52:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:39 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 6284/234936 objects degraded (2.675%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:39.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:39 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 6284/234936 objects degraded (2.675%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:39 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 6284/234936 objects degraded (2.675%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:52:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:40 smithi055 ceph-mon[86906]: pgmap v219: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 6091/227994 objects degraded (2.672%); 3.3 MiB/s, 73 objects/s recovering 2024-04-26T04:52:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:40 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3671914668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:40 smithi038 ceph-mon[93172]: pgmap v219: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 6091/227994 objects degraded (2.672%); 3.3 MiB/s, 73 objects/s recovering 2024-04-26T04:52:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:40 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3671914668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:40 smithi038 ceph-mon[96929]: pgmap v219: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 6091/227994 objects degraded (2.672%); 3.3 MiB/s, 73 objects/s recovering 2024-04-26T04:52:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:40 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3671914668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:42 smithi055 ceph-mon[86906]: pgmap v220: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 6091/223779 objects degraded (2.722%); 1.9 MiB/s, 48 objects/s recovering 2024-04-26T04:52:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:42 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3108683140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:42 smithi038 ceph-mon[93172]: pgmap v220: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 6091/223779 objects degraded (2.722%); 1.9 MiB/s, 48 objects/s recovering 2024-04-26T04:52:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:42 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3108683140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:42 smithi038 ceph-mon[96929]: pgmap v220: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 6091/223779 objects degraded (2.722%); 1.9 MiB/s, 48 objects/s recovering 2024-04-26T04:52:42.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:42 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3108683140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:44 smithi055 ceph-mon[86906]: pgmap v221: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6091/223779 objects degraded (2.722%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:52:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 6091/223779 objects degraded (2.722%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:44.660 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:44 smithi038 ceph-mon[93172]: pgmap v221: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6091/223779 objects degraded (2.722%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:52:44.660 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 6091/223779 objects degraded (2.722%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:44.660 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:44 smithi038 ceph-mon[96929]: pgmap v221: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 6091/223779 objects degraded (2.722%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:52:44.660 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 6091/223779 objects degraded (2.722%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:45 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1423007958' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:45.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:45 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1423007958' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:45.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:45 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1423007958' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:46 smithi055 ceph-mon[86906]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 906 op/s; 6007/213153 objects degraded (2.818%); 2.3 MiB/s, 45 objects/s recovering 2024-04-26T04:52:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-mon[93172]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 906 op/s; 6007/213153 objects degraded (2.818%); 2.3 MiB/s, 45 objects/s recovering 2024-04-26T04:52:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-mon[96929]: pgmap v222: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 906 op/s; 6007/213153 objects degraded (2.818%); 2.3 MiB/s, 45 objects/s recovering 2024-04-26T04:52:46.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:46.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:52:46] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.43.0" 2024-04-26T04:52:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:47 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/776224864' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/776224864' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:47.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:52:47.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/776224864' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:47.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:47.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:48.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:52:48.557+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:52:48.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[93172]: pgmap v223: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 6007/213153 objects degraded (2.818%); 1.4 MiB/s, 31 objects/s recovering 2024-04-26T04:52:48.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:48.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:48.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[96929]: pgmap v223: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 6007/213153 objects degraded (2.818%); 1.4 MiB/s, 31 objects/s recovering 2024-04-26T04:52:48.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:48.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:48 smithi055 ceph-mon[86906]: pgmap v223: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 6007/213153 objects degraded (2.818%); 1.4 MiB/s, 31 objects/s recovering 2024-04-26T04:52:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:52:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:52:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:52:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 6007/206640 objects degraded (2.907%); 1.5 MiB/s, 49 objects/s recovering 2024-04-26T04:52:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 6007/206640 objects degraded (2.907%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 6007/206640 objects degraded (2.907%); 1.5 MiB/s, 49 objects/s recovering 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:49.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:49 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 6007/206640 objects degraded (2.907%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: pgmap v224: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 6007/206640 objects degraded (2.907%); 1.5 MiB/s, 49 objects/s recovering 2024-04-26T04:52:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:52:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:52:49.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:49.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:49 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 6007/206640 objects degraded (2.907%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:50 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:50 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:52:50.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:50 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/989628371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:50 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:50 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:52:50.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:50 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/989628371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:50 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:52:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:50 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:52:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:50 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/989628371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:51.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:51 smithi038 ceph-mon[93172]: pgmap v225: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 6007/202767 objects degraded (2.963%); 459 KiB/s, 32 objects/s recovering 2024-04-26T04:52:51.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:51 smithi038 ceph-mon[96929]: pgmap v225: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 6007/202767 objects degraded (2.963%); 459 KiB/s, 32 objects/s recovering 2024-04-26T04:52:51.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:51 smithi055 ceph-mon[86906]: pgmap v225: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 6007/202767 objects degraded (2.963%); 459 KiB/s, 32 objects/s recovering 2024-04-26T04:52:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:52 smithi038 ceph-mon[93172]: pgmap v226: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 6007/202767 objects degraded (2.963%); 459 KiB/s, 32 objects/s recovering 2024-04-26T04:52:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:52 smithi038 ceph-mon[96929]: pgmap v226: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 6007/202767 objects degraded (2.963%); 459 KiB/s, 32 objects/s recovering 2024-04-26T04:52:52.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:52 smithi055 ceph-mon[86906]: pgmap v226: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 6007/202767 objects degraded (2.963%); 459 KiB/s, 32 objects/s recovering 2024-04-26T04:52:53.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:53 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/18471749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:53.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:53 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/18471749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:53.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:53 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/18471749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:54.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:54 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 6007/202767 objects degraded (2.963%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:54.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:54 smithi038 ceph-mon[96929]: pgmap v227: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 6007/191730 objects degraded (3.133%); 459 KiB/s, 50 objects/s recovering 2024-04-26T04:52:54.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:54 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 6007/202767 objects degraded (2.963%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:54.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:54 smithi038 ceph-mon[93172]: pgmap v227: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 6007/191730 objects degraded (3.133%); 459 KiB/s, 50 objects/s recovering 2024-04-26T04:52:54.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:54 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 6007/202767 objects degraded (2.963%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:54.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:54 smithi055 ceph-mon[86906]: pgmap v227: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 6007/191730 objects degraded (3.133%); 459 KiB/s, 50 objects/s recovering 2024-04-26T04:52:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:55 smithi038 ceph-mon[93172]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:52:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:55 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1303043991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:55.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:55 smithi038 ceph-mon[96929]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:52:55.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:55 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1303043991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:55.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:55 smithi055 ceph-mon[86906]: osdmap e98: 8 total, 8 up, 8 in 2024-04-26T04:52:55.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:55 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1303043991' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:56 smithi038 ceph-mon[96929]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T04:52:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:56 smithi038 ceph-mon[96929]: pgmap v230: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 6007/191730 objects degraded (3.133%); 0 B/s, 25 objects/s recovering 2024-04-26T04:52:56.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:56 smithi038 ceph-mon[93172]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T04:52:56.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:56 smithi038 ceph-mon[93172]: pgmap v230: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 6007/191730 objects degraded (3.133%); 0 B/s, 25 objects/s recovering 2024-04-26T04:52:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:56 smithi055 ceph-mon[86906]: osdmap e99: 8 total, 8 up, 8 in 2024-04-26T04:52:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:56 smithi055 ceph-mon[86906]: pgmap v230: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 6007/191730 objects degraded (3.133%); 0 B/s, 25 objects/s recovering 2024-04-26T04:52:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:52:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:52:56] "GET /metrics HTTP/1.1" 200 36266 "" "Prometheus/2.43.0" 2024-04-26T04:52:57.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:57 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1068826853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:57.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:57 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1068826853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:57 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1068826853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:52:58.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:58 smithi038 ceph-mon[93172]: pgmap v231: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 6007/185622 objects degraded (3.236%); 0 B/s, 28 objects/s recovering 2024-04-26T04:52:58.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:52:58 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 6007/191730 objects degraded (3.133%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:58 smithi038 ceph-mon[96929]: pgmap v231: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 6007/185622 objects degraded (3.236%); 0 B/s, 28 objects/s recovering 2024-04-26T04:52:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:52:58 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 6007/191730 objects degraded (3.133%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:52:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:58 smithi055 ceph-mon[86906]: pgmap v231: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 6007/185622 objects degraded (3.236%); 0 B/s, 28 objects/s recovering 2024-04-26T04:52:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:52:58 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 6007/191730 objects degraded (3.133%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:00 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/380560646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:00.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/380560646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:00.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/380560646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:01.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:01 smithi055 ceph-mon[86906]: pgmap v232: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 5228/182154 objects degraded (2.870%); 2.2 MiB/s, 63 objects/s recovering 2024-04-26T04:53:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:01 smithi038 ceph-mon[93172]: pgmap v232: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 5228/182154 objects degraded (2.870%); 2.2 MiB/s, 63 objects/s recovering 2024-04-26T04:53:01.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:01 smithi038 ceph-mon[96929]: pgmap v232: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 5228/182154 objects degraded (2.870%); 2.2 MiB/s, 63 objects/s recovering 2024-04-26T04:53:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:02.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:03 smithi055 ceph-mon[86906]: pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 5228/182154 objects degraded (2.870%); 2.2 MiB/s, 37 objects/s recovering 2024-04-26T04:53:03.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:03 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/136661594' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:03.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:03 smithi038 ceph-mon[93172]: pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 5228/182154 objects degraded (2.870%); 2.2 MiB/s, 37 objects/s recovering 2024-04-26T04:53:03.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:03 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/136661594' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:03.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:03 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:53:03.574+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:03.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:03 smithi038 ceph-mon[96929]: pgmap v233: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 398 op/s; 5228/182154 objects degraded (2.870%); 2.2 MiB/s, 37 objects/s recovering 2024-04-26T04:53:03.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:03 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/136661594' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:04 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:04 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 5228/182154 objects degraded (2.870%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:04.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:04 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:04.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:04 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 5228/182154 objects degraded (2.870%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:04.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:04 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:04.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:04 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 5228/182154 objects degraded (2.870%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:05 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:05 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:05 smithi055 ceph-mon[86906]: pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 5055/170889 objects degraded (2.958%); 2.9 MiB/s, 47 objects/s recovering 2024-04-26T04:53:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:05 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:05 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:05 smithi038 ceph-mon[93172]: pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 5055/170889 objects degraded (2.958%); 2.9 MiB/s, 47 objects/s recovering 2024-04-26T04:53:05.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:05 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:05.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:05 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:05.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:05 smithi038 ceph-mon[96929]: pgmap v234: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 5055/170889 objects degraded (2.958%); 2.9 MiB/s, 47 objects/s recovering 2024-04-26T04:53:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:06 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/864433227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:06 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/864433227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:06.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:06 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/864433227' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:53:06] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-04-26T04:53:07.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:07 smithi038 ceph-mon[93172]: pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 5055/170889 objects degraded (2.958%); 2.6 MiB/s, 43 objects/s recovering 2024-04-26T04:53:07.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:07 smithi038 ceph-mon[96929]: pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 5055/170889 objects degraded (2.958%); 2.6 MiB/s, 43 objects/s recovering 2024-04-26T04:53:07.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:07 smithi055 ceph-mon[86906]: pgmap v235: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 5055/170889 objects degraded (2.958%); 2.6 MiB/s, 43 objects/s recovering 2024-04-26T04:53:08.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:08 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3591027107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:08.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:08 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3591027107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:08.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:08 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3591027107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:09 smithi038 ceph-mon[93172]: pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 5055/165384 objects degraded (3.057%); 2.4 MiB/s, 39 objects/s recovering 2024-04-26T04:53:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 5055/165384 objects degraded (3.057%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:09 smithi038 ceph-mon[96929]: pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 5055/165384 objects degraded (3.057%); 2.4 MiB/s, 39 objects/s recovering 2024-04-26T04:53:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 5055/165384 objects degraded (3.057%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:09.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:09 smithi055 ceph-mon[86906]: pgmap v236: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 5055/165384 objects degraded (3.057%); 2.4 MiB/s, 39 objects/s recovering 2024-04-26T04:53:09.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 5055/165384 objects degraded (3.057%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:10 smithi038 ceph-mon[93172]: pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 4870/159612 objects degraded (3.051%); 3.3 MiB/s, 53 objects/s recovering 2024-04-26T04:53:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:10 smithi038 ceph-mon[96929]: pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 4870/159612 objects degraded (3.051%); 3.3 MiB/s, 53 objects/s recovering 2024-04-26T04:53:10.757 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:10 smithi055 ceph-mon[86906]: pgmap v237: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 4870/159612 objects degraded (3.051%); 3.3 MiB/s, 53 objects/s recovering 2024-04-26T04:53:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:11 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3655919082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:11 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3655919082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:11.767 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:11 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3655919082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:12 smithi038 ceph-mon[93172]: pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 4870/159612 objects degraded (3.051%); 1.9 MiB/s, 29 objects/s recovering 2024-04-26T04:53:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:12 smithi038 ceph-mon[96929]: pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 4870/159612 objects degraded (3.051%); 1.9 MiB/s, 29 objects/s recovering 2024-04-26T04:53:12.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:12 smithi055 ceph-mon[86906]: pgmap v238: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 4870/159612 objects degraded (3.051%); 1.9 MiB/s, 29 objects/s recovering 2024-04-26T04:53:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:13 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4238120030' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:13.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:13 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4238120030' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:13 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4238120030' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 4870/159612 objects degraded (3.051%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:14 smithi038 ceph-mon[93172]: pgmap v239: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 4822/148116 objects degraded (3.256%); 2.1 MiB/s, 46 objects/s recovering 2024-04-26T04:53:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 4870/159612 objects degraded (3.051%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:14 smithi038 ceph-mon[96929]: pgmap v239: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 4822/148116 objects degraded (3.256%); 2.1 MiB/s, 46 objects/s recovering 2024-04-26T04:53:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 4870/159612 objects degraded (3.051%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:14 smithi055 ceph-mon[86906]: pgmap v239: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 4822/148116 objects degraded (3.256%); 2.1 MiB/s, 46 objects/s recovering 2024-04-26T04:53:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:16 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1753003600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:16 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1753003600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:16 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1753003600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:53:16] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-04-26T04:53:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:17 smithi038 ceph-mon[93172]: pgmap v240: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4822/148116 objects degraded (3.256%); 1.2 MiB/s, 31 objects/s recovering 2024-04-26T04:53:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:17.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:17 smithi038 ceph-mon[96929]: pgmap v240: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4822/148116 objects degraded (3.256%); 1.2 MiB/s, 31 objects/s recovering 2024-04-26T04:53:17.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:17 smithi055 ceph-mon[86906]: pgmap v240: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4822/148116 objects degraded (3.256%); 1.2 MiB/s, 31 objects/s recovering 2024-04-26T04:53:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:18.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:18 smithi038 ceph-mon[96929]: pgmap v241: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 4822/142380 objects degraded (3.387%); 1.2 MiB/s, 31 objects/s recovering 2024-04-26T04:53:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:18 smithi038 ceph-mon[93172]: pgmap v241: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 4822/142380 objects degraded (3.387%); 1.2 MiB/s, 31 objects/s recovering 2024-04-26T04:53:18.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:18 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:53:18.594+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:18 smithi055 ceph-mon[86906]: pgmap v241: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 4822/142380 objects degraded (3.387%); 1.2 MiB/s, 31 objects/s recovering 2024-04-26T04:53:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 4822/142380 objects degraded (3.387%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2736156835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 4822/142380 objects degraded (3.387%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:19 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2736156835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:19 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:19.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:19 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:19.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:19 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 4822/142380 objects degraded (3.387%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:19.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:19 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2736156835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:20 smithi038 ceph-mon[93172]: pgmap v242: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4822/136770 objects degraded (3.526%); 1.2 MiB/s, 49 objects/s recovering 2024-04-26T04:53:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:20 smithi038 ceph-mon[96929]: pgmap v242: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4822/136770 objects degraded (3.526%); 1.2 MiB/s, 49 objects/s recovering 2024-04-26T04:53:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:20 smithi055 ceph-mon[86906]: pgmap v242: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 4822/136770 objects degraded (3.526%); 1.2 MiB/s, 49 objects/s recovering 2024-04-26T04:53:21.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:21 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3108679903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:21 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3108679903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:21.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:21 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3108679903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:22 smithi055 ceph-mon[86906]: pgmap v243: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 4822/136770 objects degraded (3.526%); 256 KiB/s, 34 objects/s recovering 2024-04-26T04:53:22.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:22 smithi038 ceph-mon[93172]: pgmap v243: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 4822/136770 objects degraded (3.526%); 256 KiB/s, 34 objects/s recovering 2024-04-26T04:53:22.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:22 smithi038 ceph-mon[96929]: pgmap v243: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 4822/136770 objects degraded (3.526%); 256 KiB/s, 34 objects/s recovering 2024-04-26T04:53:23.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:23 smithi055 ceph-mon[86906]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T04:53:23.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:23 smithi055 ceph-mon[86906]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T04:53:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:23 smithi038 ceph-mon[93172]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T04:53:23.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:23 smithi038 ceph-mon[93172]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T04:53:23.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:23 smithi038 ceph-mon[96929]: osdmap e100: 8 total, 8 up, 8 in 2024-04-26T04:53:23.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:23 smithi038 ceph-mon[96929]: osdmap e101: 8 total, 8 up, 8 in 2024-04-26T04:53:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 4822/136770 objects degraded (3.526%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:24 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1761032401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:24 smithi055 ceph-mon[86906]: pgmap v246: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 4260/126276 objects degraded (3.374%); 376 KiB/s, 49 objects/s recovering 2024-04-26T04:53:24.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 4822/136770 objects degraded (3.526%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:24.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:24 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1761032401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:24.875 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:24 smithi038 ceph-mon[93172]: pgmap v246: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 4260/126276 objects degraded (3.374%); 376 KiB/s, 49 objects/s recovering 2024-04-26T04:53:24.875 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 4822/136770 objects degraded (3.526%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:24.875 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:24 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1761032401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:24.875 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:24 smithi038 ceph-mon[96929]: pgmap v246: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 909 op/s; 4260/126276 objects degraded (3.374%); 376 KiB/s, 49 objects/s recovering 2024-04-26T04:53:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2084214685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:26.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2084214685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:26.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2084214685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:53:26] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-04-26T04:53:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:27 smithi055 ceph-mon[86906]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 4260/126276 objects degraded (3.374%); 376 KiB/s, 49 objects/s recovering 2024-04-26T04:53:27.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:27 smithi038 ceph-mon[93172]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 4260/126276 objects degraded (3.374%); 376 KiB/s, 49 objects/s recovering 2024-04-26T04:53:27.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:27 smithi038 ceph-mon[96929]: pgmap v247: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 4260/126276 objects degraded (3.374%); 376 KiB/s, 49 objects/s recovering 2024-04-26T04:53:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:28 smithi038 ceph-mon[93172]: pgmap v248: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3916/121815 objects degraded (3.215%); 3.0 MiB/s, 64 objects/s recovering 2024-04-26T04:53:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:28 smithi038 ceph-mon[96929]: pgmap v248: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3916/121815 objects degraded (3.215%); 3.0 MiB/s, 64 objects/s recovering 2024-04-26T04:53:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:28 smithi055 ceph-mon[86906]: pgmap v248: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3916/121815 objects degraded (3.215%); 3.0 MiB/s, 64 objects/s recovering 2024-04-26T04:53:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:29 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 3916/121815 objects degraded (3.215%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:29 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3140139055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:29 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 3916/121815 objects degraded (3.215%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:29 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3140139055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:29 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 3916/121815 objects degraded (3.215%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:29 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3140139055' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:30 smithi055 ceph-mon[86906]: pgmap v249: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 3916/116541 objects degraded (3.360%); 3.0 MiB/s, 64 objects/s recovering 2024-04-26T04:53:30.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:30 smithi038 ceph-mon[93172]: pgmap v249: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 3916/116541 objects degraded (3.360%); 3.0 MiB/s, 64 objects/s recovering 2024-04-26T04:53:30.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:30 smithi038 ceph-mon[96929]: pgmap v249: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 3916/116541 objects degraded (3.360%); 3.0 MiB/s, 64 objects/s recovering 2024-04-26T04:53:31.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/184212320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:31.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/184212320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:31.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/184212320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:32 smithi055 ceph-mon[86906]: pgmap v250: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3916/116541 objects degraded (3.360%); 2.4 MiB/s, 52 objects/s recovering 2024-04-26T04:53:32.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:32.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:32 smithi038 ceph-mon[93172]: pgmap v250: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3916/116541 objects degraded (3.360%); 2.4 MiB/s, 52 objects/s recovering 2024-04-26T04:53:32.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:32.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:32 smithi038 ceph-mon[96929]: pgmap v250: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 3916/116541 objects degraded (3.360%); 2.4 MiB/s, 52 objects/s recovering 2024-04-26T04:53:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:33 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 3916/116541 objects degraded (3.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:33.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:33 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:33.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:33 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:53:33.612+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:33.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:33 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 3916/116541 objects degraded (3.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:33.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:33 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:34.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:33 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 3916/116541 objects degraded (3.360%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:34.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:33 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2657522894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[93172]: pgmap v251: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3702/105297 objects degraded (3.516%); 3.2 MiB/s, 63 objects/s recovering 2024-04-26T04:53:34.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:34.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:34.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2657522894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:34.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:34 smithi038 ceph-mon[96929]: pgmap v251: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3702/105297 objects degraded (3.516%); 3.2 MiB/s, 63 objects/s recovering 2024-04-26T04:53:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:34 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:34 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:53:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:34 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2657522894' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:35.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:34 smithi055 ceph-mon[86906]: pgmap v251: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 3702/105297 objects degraded (3.516%); 3.2 MiB/s, 63 objects/s recovering 2024-04-26T04:53:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:36 smithi038 ceph-mon[93172]: pgmap v252: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 3702/105297 objects degraded (3.516%); 2.9 MiB/s, 45 objects/s recovering 2024-04-26T04:53:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:36 smithi038 ceph-mon[96929]: pgmap v252: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 3702/105297 objects degraded (3.516%); 2.9 MiB/s, 45 objects/s recovering 2024-04-26T04:53:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:36 smithi055 ceph-mon[86906]: pgmap v252: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 3702/105297 objects degraded (3.516%); 2.9 MiB/s, 45 objects/s recovering 2024-04-26T04:53:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:53:36] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-04-26T04:53:37.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:37 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/801414031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:37.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:37 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/801414031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:37.858 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:37 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/801414031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:38 smithi055 ceph-mon[86906]: pgmap v253: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 3487/99669 objects degraded (3.499%); 4.0 MiB/s, 63 objects/s recovering 2024-04-26T04:53:38.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:38 smithi038 ceph-mon[93172]: pgmap v253: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 3487/99669 objects degraded (3.499%); 4.0 MiB/s, 63 objects/s recovering 2024-04-26T04:53:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:38 smithi038 ceph-mon[96929]: pgmap v253: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 3487/99669 objects degraded (3.499%); 4.0 MiB/s, 63 objects/s recovering 2024-04-26T04:53:39.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:39 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 3487/99669 objects degraded (3.499%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:39.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:39 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/960804468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:39 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 3487/99669 objects degraded (3.499%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:39 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/960804468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:39.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:39 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 3487/99669 objects degraded (3.499%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:39.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:39 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/960804468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:40.757 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:40 smithi055 ceph-mon[86906]: pgmap v254: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3487/94131 objects degraded (3.704%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:53:40.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:40 smithi038 ceph-mon[93172]: pgmap v254: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3487/94131 objects degraded (3.704%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:53:40.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:40 smithi038 ceph-mon[96929]: pgmap v254: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3487/94131 objects degraded (3.704%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:53:42.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:42 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2367360469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:42 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2367360469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:42 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2367360469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:43 smithi055 ceph-mon[86906]: pgmap v255: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3487/94131 objects degraded (3.704%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:53:43.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:43 smithi038 ceph-mon[93172]: pgmap v255: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3487/94131 objects degraded (3.704%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:53:43.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:43 smithi038 ceph-mon[96929]: pgmap v255: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 3487/94131 objects degraded (3.704%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:53:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 3487/94131 objects degraded (3.704%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 3487/94131 objects degraded (3.704%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 3487/94131 objects degraded (3.704%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:45.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:45 smithi055 ceph-mon[86906]: pgmap v256: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 3438/82959 objects degraded (4.144%); 2.5 MiB/s, 56 objects/s recovering 2024-04-26T04:53:45.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:45 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/130354687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:45.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:45 smithi038 ceph-mon[93172]: pgmap v256: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 3438/82959 objects degraded (4.144%); 2.5 MiB/s, 56 objects/s recovering 2024-04-26T04:53:45.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:45 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/130354687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:45.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:45 smithi038 ceph-mon[96929]: pgmap v256: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 932 op/s; 3438/82959 objects degraded (4.144%); 2.5 MiB/s, 56 objects/s recovering 2024-04-26T04:53:45.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:45 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/130354687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:46 smithi038 ceph-mon[93172]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 3438/82959 objects degraded (4.144%); 1.4 MiB/s, 38 objects/s recovering 2024-04-26T04:53:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:46 smithi038 ceph-mon[96929]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 3438/82959 objects degraded (4.144%); 1.4 MiB/s, 38 objects/s recovering 2024-04-26T04:53:46.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:46 smithi055 ceph-mon[86906]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 3438/82959 objects degraded (4.144%); 1.4 MiB/s, 38 objects/s recovering 2024-04-26T04:53:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:53:46] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-04-26T04:53:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:47 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2085969727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:47.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:47.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:47 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2085969727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:47.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:53:47.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:47 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2085969727' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:48.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:48 smithi038 ceph-mon[93172]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 3438/77175 objects degraded (4.455%); 1.4 MiB/s, 62 objects/s recovering 2024-04-26T04:53:48.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:48 smithi038 ceph-mon[96929]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 3438/77175 objects degraded (4.455%); 1.4 MiB/s, 62 objects/s recovering 2024-04-26T04:53:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:48 smithi055 ceph-mon[86906]: pgmap v258: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 3438/77175 objects degraded (4.455%); 1.4 MiB/s, 62 objects/s recovering 2024-04-26T04:53:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:49 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 3438/77175 objects degraded (4.455%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:49.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:49 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 3438/77175 objects degraded (4.455%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:49.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:49.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:49 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 3438/77175 objects degraded (4.455%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:49.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:50.813 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:50 smithi055 ceph-mon[86906]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:53:50.813 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:50 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2472846358' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:50.813 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:50 smithi055 ceph-mon[86906]: pgmap v260: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3438/71628 objects degraded (4.800%); 313 KiB/s, 53 objects/s recovering 2024-04-26T04:53:50.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:50 smithi038 ceph-mon[93172]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:53:50.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:50 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2472846358' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:50.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:50 smithi038 ceph-mon[93172]: pgmap v260: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3438/71628 objects degraded (4.800%); 313 KiB/s, 53 objects/s recovering 2024-04-26T04:53:50.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:50 smithi038 ceph-mon[96929]: osdmap e102: 8 total, 8 up, 8 in 2024-04-26T04:53:50.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:50 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2472846358' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:50.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:50 smithi038 ceph-mon[96929]: pgmap v260: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 3438/71628 objects degraded (4.800%); 313 KiB/s, 53 objects/s recovering 2024-04-26T04:53:51.825 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:51 smithi038 ceph-mon[93172]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:53:51.825 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:51 smithi038 ceph-mon[96929]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:53:51.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:51 smithi055 ceph-mon[86906]: osdmap e103: 8 total, 8 up, 8 in 2024-04-26T04:53:52.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:52 smithi055 ceph-mon[86906]: pgmap v262: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 3438/71628 objects degraded (4.800%); 0 B/s, 35 objects/s recovering 2024-04-26T04:53:52.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:52 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1514694251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:52 smithi038 ceph-mon[93172]: pgmap v262: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 3438/71628 objects degraded (4.800%); 0 B/s, 35 objects/s recovering 2024-04-26T04:53:52.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:52 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1514694251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:52.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:52 smithi038 ceph-mon[96929]: pgmap v262: 89 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 471 op/s; 3438/71628 objects degraded (4.800%); 0 B/s, 35 objects/s recovering 2024-04-26T04:53:52.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:52 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1514694251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:53.785 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:53 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 3438/71628 objects degraded (4.800%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:53.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:53 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 3438/71628 objects degraded (4.800%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:53.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:53 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 3438/71628 objects degraded (4.800%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:54.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:54 smithi038 ceph-mon[96929]: pgmap v263: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 2629/62793 objects degraded (4.187%); 2.4 MiB/s, 77 objects/s recovering 2024-04-26T04:53:54.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:54.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:54 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:54 smithi038 ceph-mon[93172]: pgmap v263: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 2629/62793 objects degraded (4.187%); 2.4 MiB/s, 77 objects/s recovering 2024-04-26T04:53:54.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:54.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:54 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:55.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:54 smithi055 ceph-mon[86906]: pgmap v263: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 2629/62793 objects degraded (4.187%); 2.4 MiB/s, 77 objects/s recovering 2024-04-26T04:53:55.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:55.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:54 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:55.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:55 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1713534882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:55.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:55 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1713534882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:56.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:55 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1713534882' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:56.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:56 smithi038 ceph-mon[93172]: pgmap v264: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2629/62793 objects degraded (4.187%); 2.4 MiB/s, 42 objects/s recovering 2024-04-26T04:53:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:56 smithi038 ceph-mon[96929]: pgmap v264: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2629/62793 objects degraded (4.187%); 2.4 MiB/s, 42 objects/s recovering 2024-04-26T04:53:57.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:56 smithi055 ceph-mon[86906]: pgmap v264: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2629/62793 objects degraded (4.187%); 2.4 MiB/s, 42 objects/s recovering 2024-04-26T04:53:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:53:56] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-04-26T04:53:58.095 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:57 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:58.095 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:57 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:58.096 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:57 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2371063434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:58.096 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:57 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:58.096 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:57 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:58.096 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:57 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2371063434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:57 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:57 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:57 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2371063434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:53:58.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:53:58.093+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:53:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: pgmap v265: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 2629/59199 objects degraded (4.441%); 2.2 MiB/s, 40 objects/s recovering 2024-04-26T04:53:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:53:59.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:53:58 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 2629/62793 objects degraded (4.187%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: pgmap v265: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 2629/59199 objects degraded (4.441%); 2.2 MiB/s, 40 objects/s recovering 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:53:59.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 2629/62793 objects degraded (4.187%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: pgmap v265: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 2629/59199 objects degraded (4.441%); 2.2 MiB/s, 40 objects/s recovering 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:53:59.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:53:58 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 2629/62793 objects degraded (4.187%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:01.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:00 smithi055 ceph-mon[86906]: pgmap v266: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 2451/52698 objects degraded (4.651%); 3.1 MiB/s, 53 objects/s recovering 2024-04-26T04:54:01.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:00 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2503440490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:01.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:00 smithi038 ceph-mon[93172]: pgmap v266: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 2451/52698 objects degraded (4.651%); 3.1 MiB/s, 53 objects/s recovering 2024-04-26T04:54:01.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2503440490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:01.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:00 smithi038 ceph-mon[96929]: pgmap v266: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 2451/52698 objects degraded (4.651%); 3.1 MiB/s, 53 objects/s recovering 2024-04-26T04:54:01.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2503440490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:03.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:02 smithi055 ceph-mon[86906]: pgmap v267: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 2451/52698 objects degraded (4.651%); 2.6 MiB/s, 45 objects/s recovering 2024-04-26T04:54:03.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3599723582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:02 smithi038 ceph-mon[93172]: pgmap v267: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 2451/52698 objects degraded (4.651%); 2.6 MiB/s, 45 objects/s recovering 2024-04-26T04:54:03.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3599723582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:03 smithi038 ceph-mon[96929]: pgmap v267: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 2451/52698 objects degraded (4.651%); 2.6 MiB/s, 45 objects/s recovering 2024-04-26T04:54:03.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:03 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3599723582' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:04 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 2451/52698 objects degraded (4.651%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:04 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 2451/52698 objects degraded (4.651%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:04.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:04 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 2451/52698 objects degraded (4.651%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:05.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:05 smithi055 ceph-mon[86906]: pgmap v268: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 963 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2451/46287 objects degraded (5.295%); 2.6 MiB/s, 44 objects/s recovering 2024-04-26T04:54:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:05 smithi038 ceph-mon[93172]: pgmap v268: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 963 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2451/46287 objects degraded (5.295%); 2.6 MiB/s, 44 objects/s recovering 2024-04-26T04:54:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:05 smithi038 ceph-mon[96929]: pgmap v268: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 963 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 2451/46287 objects degraded (5.295%); 2.6 MiB/s, 44 objects/s recovering 2024-04-26T04:54:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:06 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2088289100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:06 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2088289100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:06 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2088289100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:07.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:07 smithi038 ceph-mon[93172]: pgmap v269: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 862 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 2249/41445 objects degraded (5.426%); 2.1 MiB/s, 32 objects/s recovering 2024-04-26T04:54:07.237 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:54:06] "GET /metrics HTTP/1.1" 200 36262 "" "Prometheus/2.43.0" 2024-04-26T04:54:07.238 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:07 smithi038 ceph-mon[96929]: pgmap v269: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 862 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 2249/41445 objects degraded (5.426%); 2.1 MiB/s, 32 objects/s recovering 2024-04-26T04:54:07.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:07 smithi055 ceph-mon[86906]: pgmap v269: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 862 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 2249/41445 objects degraded (5.426%); 2.1 MiB/s, 32 objects/s recovering 2024-04-26T04:54:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:09 smithi055 ceph-mon[86906]: pgmap v270: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 775 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2249/37257 objects degraded (6.036%); 2.1 MiB/s, 32 objects/s recovering 2024-04-26T04:54:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:09 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2371262193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:09.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 2249/37257 objects degraded (6.036%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:09 smithi038 ceph-mon[93172]: pgmap v270: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 775 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2249/37257 objects degraded (6.036%); 2.1 MiB/s, 32 objects/s recovering 2024-04-26T04:54:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:09 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2371262193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 2249/37257 objects degraded (6.036%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:09 smithi038 ceph-mon[96929]: pgmap v270: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 775 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 2249/37257 objects degraded (6.036%); 2.1 MiB/s, 32 objects/s recovering 2024-04-26T04:54:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:09 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2371262193' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 2249/37257 objects degraded (6.036%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:11 smithi055 ceph-mon[86906]: pgmap v271: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 627 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 2221/30144 objects degraded (7.368%); 2.1 MiB/s, 50 objects/s recovering 2024-04-26T04:54:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:11 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1926126065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:11 smithi038 ceph-mon[93172]: pgmap v271: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 627 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 2221/30144 objects degraded (7.368%); 2.1 MiB/s, 50 objects/s recovering 2024-04-26T04:54:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:11 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1926126065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:11.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:11 smithi038 ceph-mon[96929]: pgmap v271: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 627 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 2221/30144 objects degraded (7.368%); 2.1 MiB/s, 50 objects/s recovering 2024-04-26T04:54:11.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:11 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1926126065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:13.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:13 smithi055 ceph-mon[86906]: pgmap v272: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 627 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 2221/30144 objects degraded (7.368%); 1.2 MiB/s, 35 objects/s recovering 2024-04-26T04:54:13.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:13 smithi038 ceph-mon[93172]: pgmap v272: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 627 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 2221/30144 objects degraded (7.368%); 1.2 MiB/s, 35 objects/s recovering 2024-04-26T04:54:13.474 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:54:13.113+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:54:13.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:13 smithi038 ceph-mon[96929]: pgmap v272: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 627 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 2221/30144 objects degraded (7.368%); 1.2 MiB/s, 35 objects/s recovering 2024-04-26T04:54:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:14 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:14 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:14 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:54:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:14 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3469550941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 2221/30144 objects degraded (7.368%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:54:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3469550941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 2221/30144 objects degraded (7.368%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:14.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:14.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:14.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:54:14.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3469550941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:14.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 2221/30144 objects degraded (7.368%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:15.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:15 smithi055 ceph-mon[86906]: pgmap v273: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 496 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 2221/23868 objects degraded (9.305%); 1.2 MiB/s, 35 objects/s recovering 2024-04-26T04:54:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:15 smithi038 ceph-mon[93172]: pgmap v273: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 496 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 2221/23868 objects degraded (9.305%); 1.2 MiB/s, 35 objects/s recovering 2024-04-26T04:54:15.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:15 smithi038 ceph-mon[96929]: pgmap v273: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 496 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 2221/23868 objects degraded (9.305%); 1.2 MiB/s, 35 objects/s recovering 2024-04-26T04:54:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:16 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2304245414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:16 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2304245414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:16 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2304245414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:17 smithi038 ceph-mon[93172]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 390 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 2221/18759 objects degraded (11.840%); 1.2 MiB/s, 55 objects/s recovering 2024-04-26T04:54:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:17 smithi038 ceph-mon[93172]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T04:54:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:54:16] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.43.0" 2024-04-26T04:54:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:17 smithi038 ceph-mon[96929]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 390 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 2221/18759 objects degraded (11.840%); 1.2 MiB/s, 55 objects/s recovering 2024-04-26T04:54:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:17 smithi038 ceph-mon[96929]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T04:54:17.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:17 smithi055 ceph-mon[86906]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 390 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 2221/18759 objects degraded (11.840%); 1.2 MiB/s, 55 objects/s recovering 2024-04-26T04:54:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:17 smithi055 ceph-mon[86906]: osdmap e104: 8 total, 8 up, 8 in 2024-04-26T04:54:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:18 smithi038 ceph-mon[93172]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T04:54:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:18 smithi038 ceph-mon[96929]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T04:54:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:18 smithi055 ceph-mon[86906]: osdmap e105: 8 total, 8 up, 8 in 2024-04-26T04:54:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:19 smithi038 ceph-mon[96929]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 320 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1473/15387 objects degraded (9.573%); 1.6 MiB/s, 55 objects/s recovering 2024-04-26T04:54:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 1473/15387 objects degraded (9.573%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:19 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1520859691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:19 smithi038 ceph-mon[93172]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 320 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1473/15387 objects degraded (9.573%); 1.6 MiB/s, 55 objects/s recovering 2024-04-26T04:54:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:19 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 1473/15387 objects degraded (9.573%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:19 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1520859691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:19 smithi055 ceph-mon[86906]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 320 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 1473/15387 objects degraded (9.573%); 1.6 MiB/s, 55 objects/s recovering 2024-04-26T04:54:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:19 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 1473/15387 objects degraded (9.573%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:19 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1520859691' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:21 smithi038 ceph-mon[93172]: pgmap v278: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 188 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 1473/9066 objects degraded (16.248%); 1.6 MiB/s, 64 objects/s recovering 2024-04-26T04:54:21.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:21 smithi038 ceph-mon[96929]: pgmap v278: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 188 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 1473/9066 objects degraded (16.248%); 1.6 MiB/s, 64 objects/s recovering 2024-04-26T04:54:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:21 smithi055 ceph-mon[86906]: pgmap v278: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 188 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 1473/9066 objects degraded (16.248%); 1.6 MiB/s, 64 objects/s recovering 2024-04-26T04:54:22.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:22 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3387233184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:22.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:22 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3387233184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:22.519 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Removed 124464 objects 2024-04-26T04:54:22.519 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Clean up completed and total clean up time :171.539 2024-04-26T04:54:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:22 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3387233184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:23.266 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-04-26T04:54:23.266 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-04-26T04:54:23.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:23 smithi055 ceph-mon[86906]: pgmap v279: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 188 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1473/9066 objects degraded (16.248%); 1.6 MiB/s, 64 objects/s recovering 2024-04-26T04:54:23.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:23 smithi038 ceph-mon[96929]: pgmap v279: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 188 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1473/9066 objects degraded (16.248%); 1.6 MiB/s, 64 objects/s recovering 2024-04-26T04:54:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:23 smithi038 ceph-mon[93172]: pgmap v279: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 188 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 1473/9066 objects degraded (16.248%); 1.6 MiB/s, 64 objects/s recovering 2024-04-26T04:54:23.551 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 1473/9066 objects degraded (16.248%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:24 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1526514614' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:24.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 1473/9066 objects degraded (16.248%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:24.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:24 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1526514614' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 1473/9066 objects degraded (16.248%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-26T04:54:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:24 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1526514614' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:25.450 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:25 smithi038 ceph-mon[93172]: pgmap v280: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 70 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 1408/3381 objects degraded (41.644%); 2.1 MiB/s, 67 objects/s recovering 2024-04-26T04:54:25.450 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:25 smithi038 ceph-mon[96929]: pgmap v280: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 70 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 1408/3381 objects degraded (41.644%); 2.1 MiB/s, 67 objects/s recovering 2024-04-26T04:54:25.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:25 smithi055 ceph-mon[86906]: pgmap v280: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 70 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 1408/3381 objects degraded (41.644%); 2.1 MiB/s, 67 objects/s recovering 2024-04-26T04:54:26.194 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_2' removed 2024-04-26T04:54:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/962532115' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:54:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-mon[93172]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:54:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2719765591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/962532115' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:54:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-mon[96929]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:54:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2719765591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/962532115' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:54:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:26 smithi055 ceph-mon[86906]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:54:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2719765591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:26.914 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-26T04:54:26.914 INFO:tasks.radosbench:Beginning radosbench... 2024-04-26T04:54:26.915 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-04-26T04:54:26.915 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-04-26T04:54:27.183 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:54:26] "GET /metrics HTTP/1.1" 200 36257 "" "Prometheus/2.43.0" 2024-04-26T04:54:27.189 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:27.437 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:27 smithi038 ceph-mon[93172]: pgmap v281: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1408/72 objects degraded (1955.556%); 1.7 MiB/s, 54 objects/s recovering 2024-04-26T04:54:27.437 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:27 smithi038 ceph-mon[93172]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:54:27.437 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:27 smithi038 ceph-mon[93172]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T04:54:27.438 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:27 smithi038 ceph-mon[96929]: pgmap v281: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1408/72 objects degraded (1955.556%); 1.7 MiB/s, 54 objects/s recovering 2024-04-26T04:54:27.438 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:27 smithi038 ceph-mon[96929]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:54:27.438 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:27 smithi038 ceph-mon[96929]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T04:54:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:27 smithi055 ceph-mon[86906]: pgmap v281: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1408/72 objects degraded (1955.556%); 1.7 MiB/s, 54 objects/s recovering 2024-04-26T04:54:27.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:27 smithi055 ceph-mon[86906]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:54:27.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:27 smithi055 ceph-mon[86906]: osdmap e106: 8 total, 8 up, 8 in 2024-04-26T04:54:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:28 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:28 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:28 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[96929]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[96929]: Upgrade: osd.1 is safe to restart 2024-04-26T04:54:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[96929]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1408/72 objects degraded (1955.556%), 2 pgs degraded, 2 pgs undersized) 2024-04-26T04:54:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/747860428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[93172]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[93172]: Upgrade: osd.1 is safe to restart 2024-04-26T04:54:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[93172]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1408/72 objects degraded (1955.556%), 2 pgs degraded, 2 pgs undersized) 2024-04-26T04:54:29.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:29 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/747860428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:29 smithi055 ceph-mon[86906]: pgmap v283: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:29 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-26T04:54:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:29 smithi055 ceph-mon[86906]: Upgrade: osd.1 is safe to restart 2024-04-26T04:54:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:29 smithi055 ceph-mon[86906]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1408/72 objects degraded (1955.556%), 2 pgs degraded, 2 pgs undersized) 2024-04-26T04:54:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:29 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/747860428' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:30.913 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_3' created 2024-04-26T04:54:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: Upgrade: Updating osd.1 2024-04-26T04:54:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:54:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: Deploying daemon osd.1 on smithi038 2024-04-26T04:54:31.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3526062930' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3408989817' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: Upgrade: Updating osd.1 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: Deploying daemon osd.1 on smithi038 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3526062930' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-26T04:54:31.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:30 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3408989817' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: Upgrade: Updating osd.1 2024-04-26T04:54:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-26T04:54:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: Deploying daemon osd.1 on smithi038 2024-04-26T04:54:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: pgmap v284: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3526062930' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-26T04:54:31.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:30 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3408989817' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:31.627 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-04-26T04:54:32.131 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3526062930' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[93172]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3526062930' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[96929]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:32.217 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:31 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3526062930' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-26T04:54:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:31 smithi055 ceph-mon[86906]: osdmap e107: 8 total, 8 up, 8 in 2024-04-26T04:54:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:31 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:31 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:33.068 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:32 smithi038 ceph-mon[93172]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T04:54:33.068 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:32 smithi038 ceph-mon[93172]: pgmap v287: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:33.068 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:33.068 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:32 smithi038 ceph-mon[96929]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T04:54:33.068 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:32 smithi038 ceph-mon[96929]: pgmap v287: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:33.068 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:32 smithi055 ceph-mon[86906]: osdmap e108: 8 total, 8 up, 8 in 2024-04-26T04:54:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:32 smithi055 ceph-mon[86906]: pgmap v287: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:33.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:34.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:33 smithi038 ceph-mon[96929]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:54:34.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:33 smithi038 ceph-mon[96929]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T04:54:34.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:33 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1706367546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:34.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:33 smithi038 ceph-mon[93172]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:54:34.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:33 smithi038 ceph-mon[93172]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T04:54:34.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:33 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1706367546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:34.225 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:33 smithi038 systemd[1]: Stopping Ceph osd.1 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:54:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:33 smithi055 ceph-mon[86906]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-26T04:54:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:33 smithi055 ceph-mon[86906]: osdmap e109: 8 total, 8 up, 8 in 2024-04-26T04:54:34.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:33 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1706367546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:35.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-mon[93172]: pgmap v289: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:35.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-mon[93172]: osd.1 marked itself down and dead 2024-04-26T04:54:35.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-mon[96929]: pgmap v289: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:35.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-mon[96929]: osd.1 marked itself down and dead 2024-04-26T04:54:35.225 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[46663]: 2024-04-26T04:54:34.940+0000 7f2539f0e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:54:35.225 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[46663]: 2024-04-26T04:54:34.940+0000 7f2539f0e700 -1 osd.1 109 *** Got signal Terminated *** 2024-04-26T04:54:35.225 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[46663]: 2024-04-26T04:54:34.940+0000 7f2539f0e700 -1 osd.1 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:54:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:34 smithi055 ceph-mon[86906]: pgmap v289: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:35.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:34 smithi055 ceph-mon[86906]: osd.1 marked itself down and dead 2024-04-26T04:54:35.916 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:35 smithi038 podman[113638]: 2024-04-26 04:54:35.591811207 +0000 UTC m=+0.732065555 container died 54e6c9519bdd1f3b31270bfdb21c6ed7f6ec8faf73a2fe941d7f01dd1afb5474 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T04:54:35.916 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:35 smithi038 podman[113638]: 2024-04-26 04:54:35.915723122 +0000 UTC m=+1.055977473 container cleanup 54e6c9519bdd1f3b31270bfdb21c6ed7f6ec8faf73a2fe941d7f01dd1afb5474 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:54:35.990 INFO:teuthology.orchestra.run.smithi038.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-04-26T04:54:36.224 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:35 smithi038 bash[113638]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1 2024-04-26T04:54:36.224 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:36 smithi038 podman[113682]: 2024-04-26 04:54:36.117852219 +0000 UTC m=+0.527613739 container remove 54e6c9519bdd1f3b31270bfdb21c6ed7f6ec8faf73a2fe941d7f01dd1afb5474 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-26T04:54:36.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[93172]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[93172]: osdmap e110: 8 total, 7 up, 8 in 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2593771178' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1134605275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[96929]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[96929]: osdmap e110: 8 total, 7 up, 8 in 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2593771178' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:36.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:35 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1134605275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:36.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:35 smithi055 ceph-mon[86906]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:54:36.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:35 smithi055 ceph-mon[86906]: osdmap e110: 8 total, 7 up, 8 in 2024-04-26T04:54:36.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:35 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2593771178' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-26T04:54:36.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:35 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1134605275' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:36.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:54:36] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.43.0" 2024-04-26T04:54:37.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-mon[93172]: pgmap v291: 89 pgs: 14 stale+active+clean, 75 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:37.329 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2593771178' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:37.330 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-mon[93172]: osdmap e111: 8 total, 7 up, 8 in 2024-04-26T04:54:37.330 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-mon[96929]: pgmap v291: 89 pgs: 14 stale+active+clean, 75 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:37.330 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2593771178' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:37.330 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:36 smithi038 ceph-mon[96929]: osdmap e111: 8 total, 7 up, 8 in 2024-04-26T04:54:37.331 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:37 smithi038 podman[113789]: 2024-04-26 04:54:37.10451588 +0000 UTC m=+0.018719660 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:37.331 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:37 smithi038 podman[113789]: 2024-04-26 04:54:37.214659002 +0000 UTC m=+0.128862812 container create 207367fb793c510f1b19f7c67b722ec13d01d80435dd1c572acd64ed4685b8a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:54:37.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:36 smithi055 ceph-mon[86906]: pgmap v291: 89 pgs: 14 stale+active+clean, 75 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:54:37.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2593771178' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-26T04:54:37.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:36 smithi055 ceph-mon[86906]: osdmap e111: 8 total, 7 up, 8 in 2024-04-26T04:54:37.630 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:37 smithi038 podman[113789]: 2024-04-26 04:54:37.433902161 +0000 UTC m=+0.348105927 container init 207367fb793c510f1b19f7c67b722ec13d01d80435dd1c572acd64ed4685b8a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:54:37.630 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:37 smithi038 podman[113789]: 2024-04-26 04:54:37.437275133 +0000 UTC m=+0.351478902 container start 207367fb793c510f1b19f7c67b722ec13d01d80435dd1c572acd64ed4685b8a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb) 2024-04-26T04:54:37.630 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:37 smithi038 podman[113789]: 2024-04-26 04:54:37.517466502 +0000 UTC m=+0.431670269 container attach 207367fb793c510f1b19f7c67b722ec13d01d80435dd1c572acd64ed4685b8a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:54:37.721 DEBUG:teuthology.orchestra.run.smithi038:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-04-26T04:54:37.724 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-26T04:54:37.724 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-26T04:54:37.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:hints = 1 2024-04-26T04:54:37.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-26T04:54:37.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object prefix: benchmark_data_smithi038_113838 2024-04-26T04:54:37.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:54:37.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 0 0 0 0 0 0 - 0 2024-04-26T04:54:37.974 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:37 smithi038 podman[113789]: 2024-04-26 04:54:37.758543236 +0000 UTC m=+0.672747002 container died 207367fb793c510f1b19f7c67b722ec13d01d80435dd1c572acd64ed4685b8a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb) 2024-04-26T04:54:38.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:37 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3143997386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:38.390 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:37 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3143997386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:38.390 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:37 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3143997386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:38.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:38 smithi038 podman[113858]: 2024-04-26 04:54:38.389693145 +0000 UTC m=+0.624665447 container remove 207367fb793c510f1b19f7c67b722ec13d01d80435dd1c572acd64ed4685b8a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T04:54:38.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:38 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-26T04:54:38.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:38 smithi038 systemd[1]: Stopped Ceph osd.1 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:54:38.725 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:38 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.1.service: Consumed 16min 56.063s CPU time. 2024-04-26T04:54:38.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 1 16 4272 4256 265.985 266 0.00450323 0.00374793 2024-04-26T04:54:39.074 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:38 smithi038 ceph-mon[93172]: pgmap v293: 89 pgs: 3 active+undersized+degraded, 9 stale+active+clean, 15 active+undersized, 62 active+clean; 4.4 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s wr, 10 op/s; 8/255 objects degraded (3.137%) 2024-04-26T04:54:39.075 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:38 smithi038 ceph-mon[93172]: Health check failed: Degraded data redundancy: 8/255 objects degraded (3.137%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:39.075 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:38 smithi038 ceph-mon[93172]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:39.075 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:38 smithi038 ceph-mon[96929]: pgmap v293: 89 pgs: 3 active+undersized+degraded, 9 stale+active+clean, 15 active+undersized, 62 active+clean; 4.4 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s wr, 10 op/s; 8/255 objects degraded (3.137%) 2024-04-26T04:54:39.075 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:38 smithi038 ceph-mon[96929]: Health check failed: Degraded data redundancy: 8/255 objects degraded (3.137%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:39.075 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:38 smithi038 ceph-mon[96929]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:39.075 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:38 smithi038 systemd[1]: Starting Ceph osd.1 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:54:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:38 smithi055 ceph-mon[86906]: pgmap v293: 89 pgs: 3 active+undersized+degraded, 9 stale+active+clean, 15 active+undersized, 62 active+clean; 4.4 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 651 KiB/s wr, 10 op/s; 8/255 objects degraded (3.137%) 2024-04-26T04:54:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:38 smithi055 ceph-mon[86906]: Health check failed: Degraded data redundancy: 8/255 objects degraded (3.137%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:38 smithi055 ceph-mon[86906]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-26T04:54:39.371 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:39 smithi038 podman[113970]: 2024-04-26 04:54:38.978820085 +0000 UTC m=+0.035450695 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:39.371 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:39 smithi038 podman[113970]: 2024-04-26 04:54:39.116568143 +0000 UTC m=+0.173198742 container create 01608430ef8aa4350f60b53f613460e7511b442b64bc88f673153c700ed78540 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:54:39.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:39 smithi038 podman[113970]: 2024-04-26 04:54:39.370638309 +0000 UTC m=+0.427268916 container init 01608430ef8aa4350f60b53f613460e7511b442b64bc88f673153c700ed78540 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:54:39.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:39 smithi038 podman[113970]: 2024-04-26 04:54:39.376884892 +0000 UTC m=+0.433515488 container start 01608430ef8aa4350f60b53f613460e7511b442b64bc88f673153c700ed78540 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True) 2024-04-26T04:54:39.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:39 smithi038 podman[113970]: 2024-04-26 04:54:39.487034024 +0000 UTC m=+0.543664630 container attach 01608430ef8aa4350f60b53f613460e7511b442b64bc88f673153c700ed78540 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T04:54:39.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 2 16 8205 8189 255.887 245.812 0.00348676 0.00390198 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate[113986]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-26T04:54:40.475 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-26T04:54:40.476 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 bash[113970]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-26T04:54:40.476 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 podman[113970]: 2024-04-26 04:54:40.273211609 +0000 UTC m=+1.329842217 container died 01608430ef8aa4350f60b53f613460e7511b442b64bc88f673153c700ed78540 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T04:54:40.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 3 16 11743 11727 244.293 221.125 0.00195452 0.00408653 2024-04-26T04:54:41.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:41 smithi038 ceph-mon[93172]: pgmap v294: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 464 op/s; 1986/9816 objects degraded (20.232%) 2024-04-26T04:54:41.219 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/325973029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:41.220 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:40 smithi038 podman[114142]: 2024-04-26 04:54:40.917647123 +0000 UTC m=+0.634866236 container remove 01608430ef8aa4350f60b53f613460e7511b442b64bc88f673153c700ed78540 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb) 2024-04-26T04:54:41.220 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:41 smithi038 ceph-mon[96929]: pgmap v294: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 464 op/s; 1986/9816 objects degraded (20.232%) 2024-04-26T04:54:41.220 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/325973029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:41 smithi055 ceph-mon[86906]: pgmap v294: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s wr, 464 op/s; 1986/9816 objects degraded (20.232%) 2024-04-26T04:54:41.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/325973029' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:41.474 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 podman[114197]: 2024-04-26 04:54:41.123100127 +0000 UTC m=+0.036574195 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:54:41.474 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 podman[114197]: 2024-04-26 04:54:41.252267547 +0000 UTC m=+0.165741604 container create 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-26T04:54:41.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 4 16 15191 15175 237.09 215.5 0.00148401 0.00419355 2024-04-26T04:54:42.224 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 podman[114197]: 2024-04-26 04:54:41.856033315 +0000 UTC m=+0.769507376 container init 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-26T04:54:42.224 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 podman[114197]: 2024-04-26 04:54:41.862043013 +0000 UTC m=+0.775517078 container start 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T04:54:42.224 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 ceph-osd[114214]: -- 172.21.15.38:0/1842237488 <== mon.0 v2:172.21.15.38:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x5624b83d5ba0 con 0x5624b83a7400 2024-04-26T04:54:42.224 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 bash[114197]: 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 2024-04-26T04:54:42.225 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:41 smithi038 systemd[1]: Started Ceph osd.1 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:54:42.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[114210]: 2024-04-26T04:54:42.420+0000 7f2ea5cb97c0 -1 Falling back to public interface 2024-04-26T04:54:42.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 5 16 18207 18191 227.368 188.5 0.0040186 0.00439245 2024-04-26T04:54:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:43 smithi055 ceph-mon[86906]: pgmap v295: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 1986/9816 objects degraded (20.232%) 2024-04-26T04:54:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:43 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2838425729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:43.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[93172]: pgmap v295: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 1986/9816 objects degraded (20.232%) 2024-04-26T04:54:43.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:43.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.448 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2838425729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:43.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[96929]: pgmap v295: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 204 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s wr, 405 op/s; 1986/9816 objects degraded (20.232%) 2024-04-26T04:54:43.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:43.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:43.449 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:43 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2838425729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:43.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 6 16 21171 21155 220.345 185.25 0.00447624 0.00453299 2024-04-26T04:54:44.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 1986/9816 objects degraded (20.232%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 1986/9816 objects degraded (20.232%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 1986/9816 objects degraded (20.232%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:44.774 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 7 16 24063 24047 214.686 180.75 0.00490179 0.00465212 2024-04-26T04:54:45.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:45 smithi055 ceph-mon[86906]: pgmap v296: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 776 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s; 4204/37293 objects degraded (11.273%) 2024-04-26T04:54:45.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:45 smithi038 ceph-mon[93172]: pgmap v296: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 776 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s; 4204/37293 objects degraded (11.273%) 2024-04-26T04:54:45.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:45 smithi038 ceph-mon[96929]: pgmap v296: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 776 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s; 4204/37293 objects degraded (11.273%) 2024-04-26T04:54:45.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 8 16 26576 26560 207.481 157.062 0.00535608 0.0048144 2024-04-26T04:54:46.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:46 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2450895977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:46.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:46 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2450895977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:46.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:46 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2450895977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:46.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 9 16 29267 29251 203.113 168.188 0.0056526 0.00491759 2024-04-26T04:54:47.091 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:54:46] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.43.0" 2024-04-26T04:54:47.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:47 smithi055 ceph-mon[86906]: pgmap v297: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 9350/61647 objects degraded (15.167%) 2024-04-26T04:54:47.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:47.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:47 smithi038 ceph-mon[93172]: pgmap v297: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 9350/61647 objects degraded (15.167%) 2024-04-26T04:54:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:47.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:47 smithi038 ceph-mon[96929]: pgmap v297: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 9350/61647 objects degraded (15.167%) 2024-04-26T04:54:47.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:47.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:54:47.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 10 16 31787 31771 198.551 157.5 0.00492797 0.00503108 2024-04-26T04:54:48.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 11 16 33801 33785 191.943 125.875 0.00454622 0.00520476 2024-04-26T04:54:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:48 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:48 smithi055 ceph-mon[86906]: pgmap v298: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 10174/79365 objects degraded (12.819%) 2024-04-26T04:54:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:48 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2825020562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:49.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:48 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 9350/61647 objects degraded (15.167%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[93172]: pgmap v298: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 10174/79365 objects degraded (12.819%) 2024-04-26T04:54:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2825020562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 9350/61647 objects degraded (15.167%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:49.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:49.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:49.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[96929]: pgmap v298: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 10174/79365 objects degraded (12.819%) 2024-04-26T04:54:49.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2825020562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:49.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:48 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 9350/61647 objects degraded (15.167%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:49.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 12 16 35947 35931 187.123 134.125 0.00751931 0.00533876 2024-04-26T04:54:50.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 13 16 37913 37897 182.179 122.875 0.00444534 0.00548293 2024-04-26T04:54:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:50 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:50 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:50 smithi038 ceph-mon[96929]: pgmap v299: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 174 MiB/s wr, 2.78k op/s; 14836/100242 objects degraded (14.800%) 2024-04-26T04:54:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:50 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:50 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:50 smithi038 ceph-mon[93172]: pgmap v299: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 174 MiB/s wr, 2.78k op/s; 14836/100242 objects degraded (14.800%) 2024-04-26T04:54:51.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:50 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:51.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:50 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:51.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:50 smithi055 ceph-mon[86906]: pgmap v299: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 174 MiB/s wr, 2.78k op/s; 14836/100242 objects degraded (14.800%) 2024-04-26T04:54:51.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[114210]: 2024-04-26T04:54:51.161+0000 7f2ea5cb97c0 -1 osd.1 109 log_to_monitors true 2024-04-26T04:54:51.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 14 16 39943 39927 178.228 126.875 0.00520903 0.00560573 2024-04-26T04:54:52.159 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/498081264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:52.159 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-mon[93172]: from='osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:54:52.159 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-mon[93172]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:54:52.160 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/498081264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:52.160 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-mon[96929]: from='osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:54:52.160 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-mon[96929]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:54:52.160 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 04:54:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[114210]: 2024-04-26T04:54:51.937+0000 7f2e97111700 -1 osd.1 109 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:54:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:51 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/498081264' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:51 smithi055 ceph-mon[86906]: from='osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:54:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:51 smithi055 ceph-mon[86906]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-26T04:54:52.775 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 15 16 42188 42172 175.7 140.312 0.00483877 0.00568663 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[93172]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[93172]: osdmap e112: 8 total, 7 up, 8 in 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[93172]: from='osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[93172]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[93172]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s; 14836/100242 objects degraded (14.800%) 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[96929]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:54:53.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[96929]: osdmap e112: 8 total, 7 up, 8 in 2024-04-26T04:54:53.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[96929]: from='osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:54:53.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[96929]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:54:53.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:52 smithi038 ceph-mon[96929]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s; 14836/100242 objects degraded (14.800%) 2024-04-26T04:54:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:52 smithi055 ceph-mon[86906]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-26T04:54:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:52 smithi055 ceph-mon[86906]: osdmap e112: 8 total, 7 up, 8 in 2024-04-26T04:54:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:52 smithi055 ceph-mon[86906]: from='osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:54:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:52 smithi055 ceph-mon[86906]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:54:53.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:52 smithi055 ceph-mon[86906]: pgmap v301: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 188 MiB/s wr, 3.01k op/s; 14836/100242 objects degraded (14.800%) 2024-04-26T04:54:53.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 16 16 42572 42556 166.218 24 0.00239965 0.00570106 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931] boot 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3688724335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14836/100242 objects degraded (14.800%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[96929]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T04:54:54.187 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:54:54.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931] boot 2024-04-26T04:54:54.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T04:54:54.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:54:54.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3688724335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:54.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14836/100242 objects degraded (14.800%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:54.188 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:53 smithi038 ceph-mon[93172]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T04:54:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:54:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: osd.1 [v2:172.21.15.38:6810/2413675931,v1:172.21.15.38:6811/2413675931] boot 2024-04-26T04:54:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: osdmap e113: 8 total, 8 up, 8 in 2024-04-26T04:54:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-26T04:54:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3688724335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14836/100242 objects degraded (14.800%), 10 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:54.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:53 smithi055 ceph-mon[86906]: osdmap e114: 8 total, 8 up, 8 in 2024-04-26T04:54:54.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 17 16 42572 42556 156.441 0 - 0.00570106 2024-04-26T04:54:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:54 smithi038 ceph-mon[93172]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s; 15494/114030 objects degraded (13.588%) 2024-04-26T04:54:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:54 smithi038 ceph-mon[93172]: osdmap e115: 8 total, 8 up, 8 in 2024-04-26T04:54:55.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:54 smithi038 ceph-mon[96929]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s; 15494/114030 objects degraded (13.588%) 2024-04-26T04:54:55.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:54 smithi038 ceph-mon[96929]: osdmap e115: 8 total, 8 up, 8 in 2024-04-26T04:54:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:54 smithi055 ceph-mon[86906]: pgmap v304: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.3 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s; 15494/114030 objects degraded (13.588%) 2024-04-26T04:54:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:54 smithi055 ceph-mon[86906]: osdmap e115: 8 total, 8 up, 8 in 2024-04-26T04:54:55.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:55 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:54:55.317+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T04:54:55.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 18 16 43384 43368 150.569 25.375 0.0132334 0.00663304 2024-04-26T04:54:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: pgmap v306: 89 pgs: 5 remapped+peering, 4 active+undersized+degraded, 21 active+undersized, 59 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 4734/125673 objects degraded (3.767%) 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T04:54:56.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:56 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2340145215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: pgmap v306: 89 pgs: 5 remapped+peering, 4 active+undersized+degraded, 21 active+undersized, 59 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 4734/125673 objects degraded (3.767%) 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2340145215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:54:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: pgmap v306: 89 pgs: 5 remapped+peering, 4 active+undersized+degraded, 21 active+undersized, 59 active+clean; 2.6 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 4734/125673 objects degraded (3.767%) 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-04-26T04:54:56.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2340145215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:56.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 19 16 44618 44602 146.703 77.125 0.0278958 0.00680829 2024-04-26T04:54:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:54:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:54:56] "GET /metrics HTTP/1.1" 200 36241 "" "Prometheus/2.43.0" 2024-04-26T04:54:57.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:54:57.776124+0000 min lat: 0.00104296 max lat: 2.05602 avg lat: 0.00696916 2024-04-26T04:54:57.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:54:57.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 20 16 45886 45870 143.33 79.25 0.0195221 0.00696916 2024-04-26T04:54:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:58 smithi038 ceph-mon[96929]: pgmap v307: 89 pgs: 5 remapped+peering, 4 active+undersized+degraded, 11 active+undersized, 69 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s; 4734/125673 objects degraded (3.767%) 2024-04-26T04:54:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:58 smithi038 ceph-mon[93172]: pgmap v307: 89 pgs: 5 remapped+peering, 4 active+undersized+degraded, 11 active+undersized, 69 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s; 4734/125673 objects degraded (3.767%) 2024-04-26T04:54:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:58 smithi055 ceph-mon[86906]: pgmap v307: 89 pgs: 5 remapped+peering, 4 active+undersized+degraded, 11 active+undersized, 69 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s; 4734/125673 objects degraded (3.767%) 2024-04-26T04:54:58.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 21 16 47026 47010 139.897 71.25 0.0058224 0.00714151 2024-04-26T04:54:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:59 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/513021435' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:54:59 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 4734/125673 objects degraded (3.767%), 4 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:59 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/513021435' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:54:59 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 4734/125673 objects degraded (3.767%), 4 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:59 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/513021435' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:54:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:54:59 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 4734/125673 objects degraded (3.767%), 4 pgs degraded (PG_DEGRADED) 2024-04-26T04:54:59.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 22 15 48320 48305 137.217 80.9375 0.0138357 0.00728265 2024-04-26T04:55:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:00 smithi038 ceph-mon[93172]: pgmap v308: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5191/132975 objects degraded (3.904%); 1.2 MiB/s, 19 objects/s recovering 2024-04-26T04:55:00.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:00 smithi038 ceph-mon[96929]: pgmap v308: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5191/132975 objects degraded (3.904%); 1.2 MiB/s, 19 objects/s recovering 2024-04-26T04:55:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:00 smithi055 ceph-mon[86906]: pgmap v308: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 5191/132975 objects degraded (3.904%); 1.2 MiB/s, 19 objects/s recovering 2024-04-26T04:55:00.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 23 16 49525 49509 134.523 75.25 0.0170127 0.00742716 2024-04-26T04:55:01.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 24 16 50677 50661 131.917 72 0.0059444 0.00757273 2024-04-26T04:55:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[93172]: pgmap v309: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 18425/140538 objects degraded (13.110%); 1.0 MiB/s, 16 objects/s recovering 2024-04-26T04:55:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/579886015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[96929]: pgmap v309: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 18425/140538 objects degraded (13.110%); 1.0 MiB/s, 16 objects/s recovering 2024-04-26T04:55:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/579886015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:02 smithi055 ceph-mon[86906]: pgmap v309: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 18425/140538 objects degraded (13.110%); 1.0 MiB/s, 16 objects/s recovering 2024-04-26T04:55:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/579886015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:02.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 25 16 51875 51859 129.635 74.875 0.012019 0.00770625 2024-04-26T04:55:03.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 26 16 53014 52998 127.387 71.1875 0.0225599 0.00784394 2024-04-26T04:55:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:04 smithi038 ceph-mon[93172]: pgmap v310: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 18425/148368 objects degraded (12.418%); 874 KiB/s, 13 objects/s recovering 2024-04-26T04:55:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:04 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 18425/148368 objects degraded (12.418%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:04 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3079028365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:04 smithi038 ceph-mon[96929]: pgmap v310: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 18425/148368 objects degraded (12.418%); 874 KiB/s, 13 objects/s recovering 2024-04-26T04:55:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:04 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 18425/148368 objects degraded (12.418%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:04.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:04 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3079028365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:04 smithi055 ceph-mon[86906]: pgmap v310: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 18425/148368 objects degraded (12.418%); 874 KiB/s, 13 objects/s recovering 2024-04-26T04:55:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:04 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 18425/148368 objects degraded (12.418%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:04 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3079028365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:04.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 27 16 54189 54173 125.389 73.4375 0.00320018 0.00796757 2024-04-26T04:55:05.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 28 16 55385 55369 123.58 74.75 0.00896005 0.00808516 2024-04-26T04:55:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:06 smithi038 ceph-mon[93172]: pgmap v311: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 18249/158427 objects degraded (11.519%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:55:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:06 smithi038 ceph-mon[96929]: pgmap v311: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 18249/158427 objects degraded (11.519%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:55:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:06 smithi055 ceph-mon[86906]: pgmap v311: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 18249/158427 objects degraded (11.519%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:55:06.776 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 29 16 56427 56411 121.564 65.125 0.00983429 0.00821717 2024-04-26T04:55:07.210 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:55:06] "GET /metrics HTTP/1.1" 200 36252 "" "Prometheus/2.43.0" 2024-04-26T04:55:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4256234162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:07.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4256234162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4256234162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:07.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 30 16 57589 57573 119.932 72.625 0.00685416 0.00833163 2024-04-26T04:55:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:08 smithi038 ceph-mon[93172]: pgmap v312: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 18249/158427 objects degraded (11.519%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:55:08.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:08 smithi038 ceph-mon[96929]: pgmap v312: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 18249/158427 objects degraded (11.519%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:55:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:08 smithi055 ceph-mon[86906]: pgmap v312: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s; 18249/158427 objects degraded (11.519%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:55:08.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 31 16 58667 58651 118.237 67.375 0.00819369 0.00845001 2024-04-26T04:55:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 18249/158427 objects degraded (11.519%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:09 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1547171020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:09.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 18249/158427 objects degraded (11.519%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:09.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:09 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1547171020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 18249/158427 objects degraded (11.519%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:09 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1547171020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:09.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 32 16 59752 59736 116.661 67.8125 0.00706594 0.00856574 2024-04-26T04:55:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:10 smithi055 ceph-mon[86906]: pgmap v313: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 18071/166974 objects degraded (10.823%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T04:55:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:10 smithi038 ceph-mon[93172]: pgmap v313: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 18071/166974 objects degraded (10.823%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T04:55:10.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:55:10.339+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:10 smithi038 ceph-mon[96929]: pgmap v313: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 18071/166974 objects degraded (10.823%); 2.5 MiB/s, 39 objects/s recovering 2024-04-26T04:55:10.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 33 16 60839 60823 115.184 67.9375 0.0417592 0.00867309 2024-04-26T04:55:11.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:11 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:11 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:11 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:11 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:11 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:11 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:11 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:11 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:11.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:11 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:11.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 34 16 61871 61855 113.693 64.5 0.0162541 0.0087893 2024-04-26T04:55:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:12 smithi055 ceph-mon[86906]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 18071/175443 objects degraded (10.300%); 1.9 MiB/s, 29 objects/s recovering 2024-04-26T04:55:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:12 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2405860951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:12 smithi038 ceph-mon[93172]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 18071/175443 objects degraded (10.300%); 1.9 MiB/s, 29 objects/s recovering 2024-04-26T04:55:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:12 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2405860951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:12 smithi038 ceph-mon[96929]: pgmap v314: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s; 18071/175443 objects degraded (10.300%); 1.9 MiB/s, 29 objects/s recovering 2024-04-26T04:55:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:12 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2405860951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:12.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 35 16 63002 62986 112.465 70.6875 0.0128469 0.00888577 2024-04-26T04:55:13.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 36 16 64123 64107 111.287 70.0625 0.0225673 0.0089797 2024-04-26T04:55:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:14 smithi055 ceph-mon[86906]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 18071/182484 objects degraded (9.903%); 1.8 MiB/s, 29 objects/s recovering 2024-04-26T04:55:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 18071/182484 objects degraded (9.903%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:14 smithi038 ceph-mon[93172]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 18071/182484 objects degraded (9.903%); 1.8 MiB/s, 29 objects/s recovering 2024-04-26T04:55:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 18071/182484 objects degraded (9.903%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:14 smithi038 ceph-mon[96929]: pgmap v315: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 18071/182484 objects degraded (9.903%); 1.8 MiB/s, 29 objects/s recovering 2024-04-26T04:55:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 18071/182484 objects degraded (9.903%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:14.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 37 16 65128 65112 109.976 62.8125 0.0190319 0.00908665 2024-04-26T04:55:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:15 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2318300509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:15 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2318300509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:15 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2318300509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:15.777 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 38 16 66220 66204 108.878 68.25 0.0190494 0.00917762 2024-04-26T04:55:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:16 smithi055 ceph-mon[86906]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 17888/191682 objects degraded (9.332%); 2.8 MiB/s, 44 objects/s recovering 2024-04-26T04:55:16.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:16 smithi038 ceph-mon[93172]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 17888/191682 objects degraded (9.332%); 2.8 MiB/s, 44 objects/s recovering 2024-04-26T04:55:16.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:16 smithi038 ceph-mon[96929]: pgmap v316: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 17888/191682 objects degraded (9.332%); 2.8 MiB/s, 44 objects/s recovering 2024-04-26T04:55:16.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 39 16 67257 67241 107.748 64.8125 0.00909978 0.00927443 2024-04-26T04:55:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:55:16] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2024-04-26T04:55:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:17 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/653428111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:17 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/653428111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:17 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/653428111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:17.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:55:17.777939+0000 min lat: 0.00104296 max lat: 2.05602 avg lat: 0.00937253 2024-04-26T04:55:17.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:55:17.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 40 16 68261 68245 106.623 62.75 0.00380831 0.00937253 2024-04-26T04:55:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:18 smithi055 ceph-mon[86906]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s; 17888/191682 objects degraded (9.332%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:55:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:18 smithi038 ceph-mon[93172]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s; 17888/191682 objects degraded (9.332%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:55:18.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:18 smithi038 ceph-mon[96929]: pgmap v317: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s; 17888/191682 objects degraded (9.332%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:55:18.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 41 16 69338 69322 105.664 67.3125 0.0100409 0.00945798 2024-04-26T04:55:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:19 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 17888/191682 objects degraded (9.332%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:19 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 17888/191682 objects degraded (9.332%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 17888/191682 objects degraded (9.332%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:19.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 42 16 70284 70268 104.556 59.125 0.0168136 0.0095572 2024-04-26T04:55:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:20 smithi055 ceph-mon[86906]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 17696/199608 objects degraded (8.865%); 2.9 MiB/s, 46 objects/s recovering 2024-04-26T04:55:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:20 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/469187089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:20 smithi038 ceph-mon[93172]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 17696/199608 objects degraded (8.865%); 2.9 MiB/s, 46 objects/s recovering 2024-04-26T04:55:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:20 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/469187089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:20 smithi038 ceph-mon[96929]: pgmap v318: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 17696/199608 objects degraded (8.865%); 2.9 MiB/s, 46 objects/s recovering 2024-04-26T04:55:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:20 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/469187089' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:20.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 43 16 71302 71286 103.604 63.625 0.00657813 0.00964522 2024-04-26T04:55:21.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 44 16 72330 72314 102.709 64.25 0.0139663 0.00973039 2024-04-26T04:55:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:22 smithi055 ceph-mon[86906]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 17696/207357 objects degraded (8.534%); 2.0 MiB/s, 31 objects/s recovering 2024-04-26T04:55:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:22 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3451165763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:22 smithi038 ceph-mon[93172]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 17696/207357 objects degraded (8.534%); 2.0 MiB/s, 31 objects/s recovering 2024-04-26T04:55:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:22 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3451165763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:22.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:22 smithi038 ceph-mon[96929]: pgmap v319: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 17696/207357 objects degraded (8.534%); 2.0 MiB/s, 31 objects/s recovering 2024-04-26T04:55:22.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:22 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3451165763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:22.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 45 16 73339 73323 101.828 63.0625 0.00889074 0.00981472 2024-04-26T04:55:23.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 46 16 74359 74343 101 63.75 0.0208392 0.00989544 2024-04-26T04:55:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:24 smithi055 ceph-mon[86906]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 17696/214029 objects degraded (8.268%); 2.0 MiB/s, 31 objects/s recovering 2024-04-26T04:55:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 17696/214029 objects degraded (8.268%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:24.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:24 smithi038 ceph-mon[93172]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 17696/214029 objects degraded (8.268%); 2.0 MiB/s, 31 objects/s recovering 2024-04-26T04:55:24.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 17696/214029 objects degraded (8.268%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:24.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:24 smithi038 ceph-mon[96929]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 17696/214029 objects degraded (8.268%); 2.0 MiB/s, 31 objects/s recovering 2024-04-26T04:55:24.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 17696/214029 objects degraded (8.268%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:24.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 47 16 75287 75271 100.085 58 0.00439861 0.00998463 2024-04-26T04:55:25.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:25 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2555616700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:25.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:55:25.364+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:25 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2555616700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:25.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:25 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2555616700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:25.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 48 16 76126 76110 99.0924 52.4375 0.0077955 0.0100852 2024-04-26T04:55:26.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[93172]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 17487/222564 objects degraded (7.857%); 3.0 MiB/s, 48 objects/s recovering 2024-04-26T04:55:26.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.726 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:26.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[96929]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 17487/222564 objects degraded (7.857%); 3.0 MiB/s, 48 objects/s recovering 2024-04-26T04:55:26.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.727 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:26.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 49 16 76992 76976 98.1746 54.125 0.0153584 0.0101787 2024-04-26T04:55:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:26 smithi055 ceph-mon[86906]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 17487/222564 objects degraded (7.857%); 3.0 MiB/s, 48 objects/s recovering 2024-04-26T04:55:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:26 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:26 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:26 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:55:26] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2024-04-26T04:55:27.778 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 50 16 77904 77888 97.351 57 0.0220944 0.0102654 2024-04-26T04:55:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:28 smithi038 ceph-mon[93172]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 17487/222564 objects degraded (7.857%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:55:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:28 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1979880816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:28.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:28 smithi038 ceph-mon[96929]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 17487/222564 objects degraded (7.857%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:55:28.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:28 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1979880816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:28.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 51 16 78905 78889 96.6688 62.5625 0.0226551 0.0103381 2024-04-26T04:55:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:28 smithi055 ceph-mon[86906]: pgmap v322: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 17487/222564 objects degraded (7.857%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:55:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:28 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1979880816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:29 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 17487/222564 objects degraded (7.857%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:29 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 17487/222564 objects degraded (7.857%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:29.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 52 16 79892 79876 95.9959 61.6875 0.00800396 0.0104106 2024-04-26T04:55:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:29 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 17487/222564 objects degraded (7.857%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:30 smithi038 ceph-mon[93172]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 17271/229302 objects degraded (7.532%); 3.2 MiB/s, 51 objects/s recovering 2024-04-26T04:55:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:30 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4079680753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:30.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:30 smithi038 ceph-mon[96929]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 17271/229302 objects degraded (7.532%); 3.2 MiB/s, 51 objects/s recovering 2024-04-26T04:55:30.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:30 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4079680753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:30.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 53 16 80861 80845 95.3273 60.5625 0.016396 0.0104843 2024-04-26T04:55:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:30 smithi055 ceph-mon[86906]: pgmap v323: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 17271/229302 objects degraded (7.532%); 3.2 MiB/s, 51 objects/s recovering 2024-04-26T04:55:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:30 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4079680753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:31.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 54 16 81827 81811 94.6799 60.375 0.0100103 0.0105554 2024-04-26T04:55:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:32 smithi038 ceph-mon[93172]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 17271/236223 objects degraded (7.311%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:55:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:32 smithi038 ceph-mon[96929]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 17271/236223 objects degraded (7.311%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:55:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:32.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 55 16 82773 82757 94.0333 59.125 0.0205614 0.0106279 2024-04-26T04:55:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:32 smithi055 ceph-mon[86906]: pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 17271/236223 objects degraded (7.311%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:55:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:33 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3649451095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:33 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3649451095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:33.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 56 16 83767 83751 93.4635 62.125 0.00720453 0.0106929 2024-04-26T04:55:33.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:33 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3649451095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:34 smithi038 ceph-mon[93172]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 17271/242802 objects degraded (7.113%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:55:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:34 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 17271/242802 objects degraded (7.113%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:34.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:34 smithi038 ceph-mon[96929]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 17271/242802 objects degraded (7.113%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:55:34.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:34 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 17271/242802 objects degraded (7.113%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:34.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 57 16 84737 84721 92.8873 60.625 0.0123751 0.0107577 2024-04-26T04:55:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:34 smithi055 ceph-mon[86906]: pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s; 17271/242802 objects degraded (7.113%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:55:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:34 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 17271/242802 objects degraded (7.113%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:35.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 58 16 85591 85575 92.206 53.375 0.0040962 0.010838 2024-04-26T04:55:36.707 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:36 smithi038 ceph-mon[93172]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 17041/250719 objects degraded (6.797%); 3.4 MiB/s, 54 objects/s recovering 2024-04-26T04:55:36.707 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/803276162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:36.707 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:36 smithi038 ceph-mon[96929]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 17041/250719 objects degraded (6.797%); 3.4 MiB/s, 54 objects/s recovering 2024-04-26T04:55:36.707 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/803276162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:36.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 59 16 86554 86538 91.6632 60.1875 0.035218 0.0109034 2024-04-26T04:55:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:36 smithi055 ceph-mon[86906]: pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 17041/250719 objects degraded (6.797%); 3.4 MiB/s, 54 objects/s recovering 2024-04-26T04:55:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/803276162' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:36.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:55:36] "GET /metrics HTTP/1.1" 200 36256 "" "Prometheus/2.43.0" 2024-04-26T04:55:37.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:55:37.779754+0000 min lat: 0.00104296 max lat: 2.05602 avg lat: 0.0109735 2024-04-26T04:55:37.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:55:37.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 60 15 87455 87440 91.0749 56.375 0.0229234 0.0109735 2024-04-26T04:55:38.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:38 smithi038 ceph-mon[93172]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 17041/250719 objects degraded (6.797%); 2.3 MiB/s, 37 objects/s recovering 2024-04-26T04:55:38.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:38 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1251398685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:38.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:38 smithi038 ceph-mon[96929]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 17041/250719 objects degraded (6.797%); 2.3 MiB/s, 37 objects/s recovering 2024-04-26T04:55:38.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:38 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1251398685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:38.779 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 61 16 88341 88325 90.4886 55.3125 0.0201056 0.0110454 2024-04-26T04:55:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:38 smithi055 ceph-mon[86906]: pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 17041/250719 objects degraded (6.797%); 2.3 MiB/s, 37 objects/s recovering 2024-04-26T04:55:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:38 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1251398685' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:39.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:39 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 17041/250719 objects degraded (6.797%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:39.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:39 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 17041/250719 objects degraded (6.797%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:39.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 62 16 89250 89234 89.9453 56.8125 0.0236477 0.0111109 2024-04-26T04:55:39.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:39 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 17041/250719 objects degraded (6.797%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:40.724 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:55:40.382+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[93172]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 16776/257784 objects degraded (6.508%); 3.7 MiB/s, 59 objects/s recovering 2024-04-26T04:55:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:40.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[96929]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 16776/257784 objects degraded (6.508%); 3.7 MiB/s, 59 objects/s recovering 2024-04-26T04:55:40.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:40 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:40.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 63 16 90084 90068 89.3449 52.125 0.00957422 0.0111868 2024-04-26T04:55:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:40 smithi055 ceph-mon[86906]: pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 16776/257784 objects degraded (6.508%); 3.7 MiB/s, 59 objects/s recovering 2024-04-26T04:55:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:40 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:40 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:55:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:40 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-26T04:55:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/559050550' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:41 smithi038 ceph-mon[93172]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 16776/264672 objects degraded (6.338%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:55:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/559050550' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:41 smithi038 ceph-mon[96929]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 16776/264672 objects degraded (6.338%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:55:41.757 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/559050550' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:41.757 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:41 smithi055 ceph-mon[86906]: pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 16776/264672 objects degraded (6.338%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:55:41.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 64 16 91006 90990 88.8492 57.625 0.0161044 0.0112489 2024-04-26T04:55:42.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 65 16 91930 91914 88.3707 57.75 0.0161341 0.0113099 2024-04-26T04:55:43.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 66 16 92810 92794 87.865 55 0.0199475 0.0113753 2024-04-26T04:55:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:44 smithi038 ceph-mon[93172]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 16776/270297 objects degraded (6.207%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:55:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4217009156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 16776/270297 objects degraded (6.207%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:44 smithi038 ceph-mon[96929]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 16776/270297 objects degraded (6.207%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:55:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4217009156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:44.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 16776/270297 objects degraded (6.207%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:44 smithi055 ceph-mon[86906]: pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 16776/270297 objects degraded (6.207%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:55:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4217009156' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 16776/270297 objects degraded (6.207%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:44.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 67 16 93740 93724 87.421 58.125 0.0114053 0.0114331 2024-04-26T04:55:45.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 68 16 94599 94583 86.9249 53.6875 0.0210407 0.0114985 2024-04-26T04:55:46.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:46 smithi038 ceph-mon[93172]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 16499/278067 objects degraded (5.933%); 4.0 MiB/s, 64 objects/s recovering 2024-04-26T04:55:46.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:46 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2913931463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:46.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:46 smithi038 ceph-mon[96929]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 16499/278067 objects degraded (5.933%); 4.0 MiB/s, 64 objects/s recovering 2024-04-26T04:55:46.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:46 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2913931463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:46 smithi055 ceph-mon[86906]: pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 16499/278067 objects degraded (5.933%); 4.0 MiB/s, 64 objects/s recovering 2024-04-26T04:55:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:46 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2913931463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:46.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 69 16 95198 95182 86.2076 37.4375 0.00399444 0.0115438 2024-04-26T04:55:47.205 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:55:46] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-04-26T04:55:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:47.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:55:47.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 70 16 95971 95955 85.6662 48.3125 0.00753723 0.0116661 2024-04-26T04:55:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:48 smithi038 ceph-mon[93172]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 16499/278067 objects degraded (5.933%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:55:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:48 smithi038 ceph-mon[96929]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 16499/278067 objects degraded (5.933%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:55:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:48 smithi055 ceph-mon[86906]: pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 16499/278067 objects degraded (5.933%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:55:48.780 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 71 16 96811 96795 85.199 52.5 0.0105537 0.0117318 2024-04-26T04:55:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:49 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/218750234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:49.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:49 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 16499/278067 objects degraded (5.933%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:49.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:49 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/218750234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:49.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:49 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 16499/278067 objects degraded (5.933%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:49.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:49 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/218750234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:49.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:49 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 16499/278067 objects degraded (5.933%), 7 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:49.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 72 16 97712 97696 84.7977 56.3125 0.0141164 0.0117867 2024-04-26T04:55:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:50 smithi055 ceph-mon[86906]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 16052/284142 objects degraded (5.649%); 5.1 MiB/s, 82 objects/s recovering 2024-04-26T04:55:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:50 smithi038 ceph-mon[93172]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 16052/284142 objects degraded (5.649%); 5.1 MiB/s, 82 objects/s recovering 2024-04-26T04:55:50.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:50 smithi038 ceph-mon[96929]: pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 16052/284142 objects degraded (5.649%); 5.1 MiB/s, 82 objects/s recovering 2024-04-26T04:55:50.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 73 16 98534 98518 84.3398 51.375 0.0393084 0.0118502 2024-04-26T04:55:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:51 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3688782702' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:51.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:51 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3688782702' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:51.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:51 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3688782702' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:51.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 74 16 99124 99108 83.6984 36.875 0.00362688 0.0118807 2024-04-26T04:55:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:52 smithi055 ceph-mon[86906]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 16052/290079 objects degraded (5.534%); 3.8 MiB/s, 60 objects/s recovering 2024-04-26T04:55:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:52 smithi055 ceph-mon[86906]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T04:55:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:52 smithi038 ceph-mon[93172]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 16052/290079 objects degraded (5.534%); 3.8 MiB/s, 60 objects/s recovering 2024-04-26T04:55:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:52 smithi038 ceph-mon[93172]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T04:55:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:52 smithi038 ceph-mon[96929]: pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 16052/290079 objects degraded (5.534%); 3.8 MiB/s, 60 objects/s recovering 2024-04-26T04:55:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:52 smithi038 ceph-mon[96929]: osdmap e116: 8 total, 8 up, 8 in 2024-04-26T04:55:52.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 75 16 99542 99526 82.9307 26.125 0.0321769 0.0120518 2024-04-26T04:55:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:53 smithi055 ceph-mon[86906]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T04:55:53.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:53 smithi038 ceph-mon[93172]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T04:55:53.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:53 smithi038 ceph-mon[96929]: osdmap e117: 8 total, 8 up, 8 in 2024-04-26T04:55:53.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 76 16 100374 100358 82.5236 52 0.0356533 0.0121112 2024-04-26T04:55:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:54 smithi055 ceph-mon[86906]: pgmap v337: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 15874/295236 objects degraded (5.377%); 4.9 MiB/s, 78 objects/s recovering 2024-04-26T04:55:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:54 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 15874/295236 objects degraded (5.377%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:54 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1588995293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:54.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:54 smithi038 ceph-mon[93172]: pgmap v337: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 15874/295236 objects degraded (5.377%); 4.9 MiB/s, 78 objects/s recovering 2024-04-26T04:55:54.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:54 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 15874/295236 objects degraded (5.377%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:54.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:54 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1588995293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:54.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:54 smithi038 ceph-mon[96929]: pgmap v337: 89 pgs: 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 15874/295236 objects degraded (5.377%); 4.9 MiB/s, 78 objects/s recovering 2024-04-26T04:55:54.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:54 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 15874/295236 objects degraded (5.377%), 6 pgs degraded (PG_DEGRADED) 2024-04-26T04:55:54.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:54 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1588995293' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:54.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 77 16 101275 101259 82.1832 56.3125 0.0134043 0.0121607 2024-04-26T04:55:55.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 78 16 102097 102081 81.7881 51.375 0.0113328 0.01222 2024-04-26T04:55:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:56 smithi055 ceph-mon[86906]: pgmap v338: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s; 15714/300594 objects degraded (5.228%); 6.1 MiB/s, 98 objects/s recovering 2024-04-26T04:55:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:56 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[96929]: pgmap v338: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s; 15714/300594 objects degraded (5.228%); 6.1 MiB/s, 98 objects/s recovering 2024-04-26T04:55:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:56.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[93172]: pgmap v338: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s; 15714/300594 objects degraded (5.228%); 6.1 MiB/s, 98 objects/s recovering 2024-04-26T04:55:56.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:55:56.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:56.727 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:56.781 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 79 16 102806 102790 81.3137 44.3125 0.0316496 0.0122923 2024-04-26T04:55:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:55:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:55:56] "GET /metrics HTTP/1.1" 200 36259 "" "Prometheus/2.43.0" 2024-04-26T04:55:57.506 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:57 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2575655163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:57.506 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:57 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:57.506 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:57 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:57.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:57 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2575655163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:57.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:57 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:57.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:57 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:57 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2575655163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:57 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:57 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:55:57.782 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:2024-04-26T04:55:57.782696+0000 min lat: 0.00104296 max lat: 2.05602 avg lat: 0.0123586 2024-04-26T04:55:57.782 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-26T04:55:57.782 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 80 16 103548 103532 80.8758 46.375 0.0302726 0.0123586 2024-04-26T04:55:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:58 smithi055 ceph-mon[86906]: pgmap v339: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 15714/301356 objects degraded (5.214%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:55:58.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:58 smithi038 ceph-mon[93172]: pgmap v339: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 15714/301356 objects degraded (5.214%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:55:58.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:58 smithi038 ceph-mon[96929]: pgmap v339: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s; 15714/301356 objects degraded (5.214%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:55:58.785 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 81 16 104315 104299 80.4665 47.9375 0.00935357 0.0124204 2024-04-26T04:55:59.598 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:59 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 15714/301356 objects degraded (5.214%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:55:59.599 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:55:59 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/70266987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:59.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:59 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 15714/301356 objects degraded (5.214%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:55:59.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:55:59 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/70266987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:59.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:59 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 15714/301356 objects degraded (5.214%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:55:59.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:55:59 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/70266987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:55:59.786 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 82 16 105113 105097 80.0934 49.875 0.0101285 0.0124792 2024-04-26T04:56:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:00 smithi055 ceph-mon[86906]: pgmap v340: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 15714/306603 objects degraded (5.125%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:56:00.639 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:00 smithi038 ceph-mon[93172]: pgmap v340: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 15714/306603 objects degraded (5.125%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:56:00.639 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:00 smithi038 ceph-mon[96929]: pgmap v340: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 15714/306603 objects degraded (5.125%); 2.6 MiB/s, 42 objects/s recovering 2024-04-26T04:56:00.787 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 83 16 105939 105923 79.7484 51.625 0.0214889 0.012533 2024-04-26T04:56:01.787 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 84 16 106644 106628 79.3235 44.0625 0.0105526 0.0126005 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[93172]: pgmap v341: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 15592/312471 objects degraded (4.990%); 2.9 MiB/s, 46 objects/s recovering 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2588268425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[96929]: pgmap v341: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 15592/312471 objects degraded (4.990%); 2.9 MiB/s, 46 objects/s recovering 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2588268425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:02 smithi055 ceph-mon[86906]: pgmap v341: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 755 op/s; 15592/312471 objects degraded (4.990%); 2.9 MiB/s, 46 objects/s recovering 2024-04-26T04:56:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2588268425' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:02.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 85 16 107397 107381 78.9439 47.0625 0.00864713 0.01266 2024-04-26T04:56:03.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 86 16 108166 108150 78.5848 48.0625 0.00530205 0.012717 2024-04-26T04:56:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[93172]: pgmap v342: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 15592/318054 objects degraded (4.902%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:56:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 15592/318054 objects degraded (4.902%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[96929]: pgmap v342: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 15592/318054 objects degraded (4.902%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:56:04.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:04.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:04.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 15592/318054 objects degraded (4.902%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:04 smithi055 ceph-mon[86906]: pgmap v342: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 15592/318054 objects degraded (4.902%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:56:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:04 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:04 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:04 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 15592/318054 objects degraded (4.902%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:04.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 87 16 108871 108855 78.188 44.0625 0.0175951 0.0127839 2024-04-26T04:56:05.193 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:56:04.762+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/247704514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/247704514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/247704514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:56:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:56:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:56:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:56:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:56:05.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:05.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 88 16 109706 109690 77.8925 52.1875 0.014591 0.0128323 2024-04-26T04:56:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-mon[93172]: pgmap v343: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 15472/323883 objects degraded (4.777%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-mon[96929]: pgmap v343: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 15472/323883 objects degraded (4.777%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:06.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:06.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:06 smithi055 ceph-mon[86906]: pgmap v343: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 15472/323883 objects degraded (4.777%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:06 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:06 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:06.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 89 16 110497 110481 77.5728 49.4375 0.0532778 0.0128847 2024-04-26T04:56:07.209 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:56:06] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.43.0" 2024-04-26T04:56:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/996519536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:07.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/996519536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/996519536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:07.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout: 90 4 111331 111327 77.2983 52.875 0.036719 0.0129307 2024-04-26T04:56:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:08 smithi038 ceph-mon[93172]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 677 op/s; 15472/323883 objects degraded (4.777%); 1.3 MiB/s, 21 objects/s recovering 2024-04-26T04:56:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:08 smithi038 ceph-mon[96929]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 677 op/s; 15472/323883 objects degraded (4.777%); 1.3 MiB/s, 21 objects/s recovering 2024-04-26T04:56:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:08 smithi055 ceph-mon[86906]: pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 42 MiB/s wr, 677 op/s; 15472/323883 objects degraded (4.777%); 1.3 MiB/s, 21 objects/s recovering 2024-04-26T04:56:08.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total time run: 90.0157 2024-04-26T04:56:08.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Total writes made: 111331 2024-04-26T04:56:08.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Write size: 65536 2024-04-26T04:56:08.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Object size: 65536 2024-04-26T04:56:08.788 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Bandwidth (MB/sec): 77.2997 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Bandwidth: 49.3957 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max bandwidth (MB/sec): 266 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min bandwidth (MB/sec): 0 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average IOPS: 1236 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev IOPS: 790.332 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max IOPS: 4256 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min IOPS: 0 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Average Latency(s): 0.0129314 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Stddev Latency(s): 0.0291345 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Max latency(s): 2.05602 2024-04-26T04:56:08.789 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Min latency(s): 0.00104296 2024-04-26T04:56:08.795 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Cleaning up (deleting benchmark objects) 2024-04-26T04:56:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 15472/323883 objects degraded (4.777%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 15472/323883 objects degraded (4.777%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 15472/323883 objects degraded (4.777%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:10 smithi055 ceph-mon[86906]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 15472/329697 objects degraded (4.693%); 1.3 MiB/s, 21 objects/s recovering 2024-04-26T04:56:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:10 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3065945381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:10 smithi038 ceph-mon[93172]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 15472/329697 objects degraded (4.693%); 1.3 MiB/s, 21 objects/s recovering 2024-04-26T04:56:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:10 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3065945381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:10 smithi038 ceph-mon[96929]: pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 15472/329697 objects degraded (4.693%); 1.3 MiB/s, 21 objects/s recovering 2024-04-26T04:56:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:10 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3065945381' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:12 smithi055 ceph-mon[86906]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 815 op/s; 15218/333333 objects degraded (4.565%); 2.7 MiB/s, 43 objects/s recovering 2024-04-26T04:56:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:12 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3369494380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:12 smithi038 ceph-mon[93172]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 815 op/s; 15218/333333 objects degraded (4.565%); 2.7 MiB/s, 43 objects/s recovering 2024-04-26T04:56:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:12 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3369494380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:12 smithi038 ceph-mon[96929]: pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 50 MiB/s wr, 815 op/s; 15218/333333 objects degraded (4.565%); 2.7 MiB/s, 43 objects/s recovering 2024-04-26T04:56:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:12 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3369494380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:14.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:14 smithi055 ceph-mon[86906]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 662 op/s; 15218/332883 objects degraded (4.572%); 2.0 MiB/s, 32 objects/s recovering 2024-04-26T04:56:14.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 15218/332883 objects degraded (4.572%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:14 smithi038 ceph-mon[93172]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 662 op/s; 15218/332883 objects degraded (4.572%); 2.0 MiB/s, 32 objects/s recovering 2024-04-26T04:56:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 15218/332883 objects degraded (4.572%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:14 smithi038 ceph-mon[96929]: pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 39 MiB/s wr, 662 op/s; 15218/332883 objects degraded (4.572%); 2.0 MiB/s, 32 objects/s recovering 2024-04-26T04:56:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 15218/332883 objects degraded (4.572%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:15 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1086142647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:15 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1086142647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:15 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1086142647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:16 smithi055 ceph-mon[86906]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 89 B/s rd, 29 MiB/s wr, 823 op/s; 15073/321759 objects degraded (4.685%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:56:16.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:16 smithi038 ceph-mon[93172]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 89 B/s rd, 29 MiB/s wr, 823 op/s; 15073/321759 objects degraded (4.685%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:56:16.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:16 smithi038 ceph-mon[96929]: pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 89 B/s rd, 29 MiB/s wr, 823 op/s; 15073/321759 objects degraded (4.685%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:56:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:56:16] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2024-04-26T04:56:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:18 smithi055 ceph-mon[86906]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 624 op/s; 15073/321759 objects degraded (4.685%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:56:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:18 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/25769572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:18 smithi038 ceph-mon[93172]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 624 op/s; 15073/321759 objects degraded (4.685%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:56:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:18 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/25769572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:18.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:18 smithi038 ceph-mon[96929]: pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 624 op/s; 15073/321759 objects degraded (4.685%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:56:18.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:18 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/25769572' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:19 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 15073/321759 objects degraded (4.685%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:19 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 15073/321759 objects degraded (4.685%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 15073/321759 objects degraded (4.685%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:20.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:19 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:56:19.783+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:20 smithi055 ceph-mon[86906]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 798 op/s; 15073/315519 objects degraded (4.777%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:56:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:20 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:20 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/414307479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:20 smithi038 ceph-mon[93172]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 798 op/s; 15073/315519 objects degraded (4.777%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:56:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:20 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:20 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/414307479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:20 smithi038 ceph-mon[96929]: pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 18 MiB/s wr, 798 op/s; 15073/315519 objects degraded (4.777%); 2.1 MiB/s, 33 objects/s recovering 2024-04-26T04:56:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:20 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:20 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/414307479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:21 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:21 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:21.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:21 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:21.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:21 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:21.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:21 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:21.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:21 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:22 smithi055 ceph-mon[86906]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 804 op/s; 14925/309459 objects degraded (4.823%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:56:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:22 smithi038 ceph-mon[93172]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 804 op/s; 14925/309459 objects degraded (4.823%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:56:22.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:22 smithi038 ceph-mon[96929]: pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 7.6 MiB/s wr, 804 op/s; 14925/309459 objects degraded (4.823%); 2.8 MiB/s, 45 objects/s recovering 2024-04-26T04:56:23.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:23 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/885173300' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:23.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:23 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/885173300' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:23 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/885173300' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:24 smithi055 ceph-mon[86906]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 133 KiB/s wr, 688 op/s; 14925/308694 objects degraded (4.835%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14925/308694 objects degraded (4.835%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:24.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:24 smithi038 ceph-mon[96929]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 133 KiB/s wr, 688 op/s; 14925/308694 objects degraded (4.835%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:24.625 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14925/308694 objects degraded (4.835%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:24.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:24 smithi038 ceph-mon[93172]: pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 133 KiB/s wr, 688 op/s; 14925/308694 objects degraded (4.835%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:24.625 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14925/308694 objects degraded (4.835%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:25.415 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:25 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1450917467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:25.415 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:25 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1450917467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:25.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:25 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1450917467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:26 smithi055 ceph-mon[86906]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 989 op/s; 14786/297273 objects degraded (4.974%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:26.723 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:26 smithi038 ceph-mon[93172]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 989 op/s; 14786/297273 objects degraded (4.974%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:26.723 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:26 smithi038 ceph-mon[96929]: pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 989 op/s; 14786/297273 objects degraded (4.974%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:26.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:56:26] "GET /metrics HTTP/1.1" 200 36264 "" "Prometheus/2.43.0" 2024-04-26T04:56:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:28 smithi038 ceph-mon[93172]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 14786/297273 objects degraded (4.974%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:28 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1558638272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:28 smithi038 ceph-mon[96929]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 14786/297273 objects degraded (4.974%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:28 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1558638272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:28 smithi055 ceph-mon[86906]: pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 14786/297273 objects degraded (4.974%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:28 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1558638272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:29 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14786/297273 objects degraded (4.974%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:29.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:29 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14786/297273 objects degraded (4.974%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:29 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14786/297273 objects degraded (4.974%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:30 smithi038 ceph-mon[93172]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 14786/291312 objects degraded (5.076%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:30 smithi038 ceph-mon[96929]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 14786/291312 objects degraded (5.076%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:30 smithi055 ceph-mon[86906]: pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 14786/291312 objects degraded (5.076%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:31.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2132421176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:31.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2132421176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:31.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2132421176' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:32 smithi038 ceph-mon[93172]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 14645/285753 objects degraded (5.125%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:32 smithi038 ceph-mon[96929]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 14645/285753 objects degraded (5.125%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:32 smithi055 ceph-mon[86906]: pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 826 op/s; 14645/285753 objects degraded (5.125%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:33 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4082044809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:33 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4082044809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:33.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:33 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4082044809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:34 smithi038 ceph-mon[93172]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 14645/284988 objects degraded (5.139%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:34 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14645/284988 objects degraded (5.139%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:34.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:34 smithi038 ceph-mon[96929]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 14645/284988 objects degraded (5.139%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:34.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:34 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14645/284988 objects degraded (5.139%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:34 smithi055 ceph-mon[86906]: pgmap v357: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 14645/284988 objects degraded (5.139%); 1.5 MiB/s, 23 objects/s recovering 2024-04-26T04:56:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:34 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14645/284988 objects degraded (5.139%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:35.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:56:34.800+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:35.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:35.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[93172]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 965 op/s; 14493/273912 objects degraded (5.291%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4219438816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[96929]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 965 op/s; 14493/273912 objects degraded (5.291%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:36.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4219438816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:36 smithi055 ceph-mon[86906]: pgmap v358: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 965 op/s; 14493/273912 objects degraded (5.291%); 2.2 MiB/s, 35 objects/s recovering 2024-04-26T04:56:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:36 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:36 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4219438816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:56:36] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.43.0" 2024-04-26T04:56:37.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:37 smithi055 ceph-mon[86906]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 14493/273912 objects degraded (5.291%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:37.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:37 smithi038 ceph-mon[93172]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 14493/273912 objects degraded (5.291%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:37.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:37 smithi038 ceph-mon[96929]: pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 14493/273912 objects degraded (5.291%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:38 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1377003767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:38.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:38 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1377003767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:38 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1377003767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:39.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:39 smithi055 ceph-mon[86906]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 14493/268026 objects degraded (5.407%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:39.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:39 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14493/273912 objects degraded (5.291%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:39 smithi038 ceph-mon[93172]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 14493/268026 objects degraded (5.407%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:39.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:39 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14493/273912 objects degraded (5.291%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:39.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:39 smithi038 ceph-mon[96929]: pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 14493/268026 objects degraded (5.407%); 1.5 MiB/s, 24 objects/s recovering 2024-04-26T04:56:39.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:39 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14493/273912 objects degraded (5.291%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:41 smithi038 ceph-mon[93172]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 14337/262152 objects degraded (5.469%); 2.3 MiB/s, 37 objects/s recovering 2024-04-26T04:56:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/342633272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:41.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:41 smithi038 ceph-mon[96929]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 14337/262152 objects degraded (5.469%); 2.3 MiB/s, 37 objects/s recovering 2024-04-26T04:56:41.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/342633272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:41 smithi055 ceph-mon[86906]: pgmap v361: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 14337/262152 objects degraded (5.469%); 2.3 MiB/s, 37 objects/s recovering 2024-04-26T04:56:42.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/342633272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:43.907 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:43 smithi055 ceph-mon[86906]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 14337/261414 objects degraded (5.484%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:56:43.908 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:43 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1124412364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:43.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:43 smithi038 ceph-mon[93172]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 14337/261414 objects degraded (5.484%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:56:43.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:43 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1124412364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:43.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:43 smithi038 ceph-mon[96929]: pgmap v362: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 14337/261414 objects degraded (5.484%); 1.6 MiB/s, 25 objects/s recovering 2024-04-26T04:56:43.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:43 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1124412364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:44.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14337/261414 objects degraded (5.484%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:44.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14337/261414 objects degraded (5.484%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:45.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14337/261414 objects degraded (5.484%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:45.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:45 smithi038 ceph-mon[93172]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 14163/250293 objects degraded (5.659%); 2.5 MiB/s, 40 objects/s recovering 2024-04-26T04:56:45.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:45 smithi038 ceph-mon[96929]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 14163/250293 objects degraded (5.659%); 2.5 MiB/s, 40 objects/s recovering 2024-04-26T04:56:46.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:45 smithi055 ceph-mon[86906]: pgmap v363: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s; 14163/250293 objects degraded (5.659%); 2.5 MiB/s, 40 objects/s recovering 2024-04-26T04:56:46.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:46 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3316247241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:46.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:56:46] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-04-26T04:56:46.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:46 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3316247241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:47.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:46 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3316247241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:47.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:47 smithi038 ceph-mon[93172]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 14163/250293 objects degraded (5.659%); 1.7 MiB/s, 27 objects/s recovering 2024-04-26T04:56:47.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:47.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:47 smithi038 ceph-mon[96929]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 14163/250293 objects degraded (5.659%); 1.7 MiB/s, 27 objects/s recovering 2024-04-26T04:56:47.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:47 smithi055 ceph-mon[86906]: pgmap v364: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 14163/250293 objects degraded (5.659%); 1.7 MiB/s, 27 objects/s recovering 2024-04-26T04:56:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:56:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:48 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 14163/250293 objects degraded (5.659%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:48 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 14163/250293 objects degraded (5.659%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:49.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:48 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 14163/250293 objects degraded (5.659%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:49 smithi055 ceph-mon[86906]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 14163/244665 objects degraded (5.789%); 1.7 MiB/s, 27 objects/s recovering 2024-04-26T04:56:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:49 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3389770020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:50.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:49 smithi038 ceph-mon[93172]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 14163/244665 objects degraded (5.789%); 1.7 MiB/s, 27 objects/s recovering 2024-04-26T04:56:50.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:49 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3389770020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:50.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:49 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:56:49.816+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:49 smithi038 ceph-mon[96929]: pgmap v365: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 14163/244665 objects degraded (5.789%); 1.7 MiB/s, 27 objects/s recovering 2024-04-26T04:56:50.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:49 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3389770020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:51.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:50 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:51.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:50 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:51.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:50 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:50 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:50 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:50 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:50 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:50 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:56:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:50 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:56:52.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:51 smithi055 ceph-mon[86906]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 13991/238797 objects degraded (5.859%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:56:52.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:51 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4108467482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:51 smithi038 ceph-mon[96929]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 13991/238797 objects degraded (5.859%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:56:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:51 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4108467482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:52.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:51 smithi038 ceph-mon[93172]: pgmap v366: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 13991/238797 objects degraded (5.859%); 2.6 MiB/s, 41 objects/s recovering 2024-04-26T04:56:52.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:51 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4108467482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:54.026 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:53 smithi038 ceph-mon[93172]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 13991/238047 objects degraded (5.877%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:56:54.026 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:53 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 13991/238047 objects degraded (5.877%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:54.026 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:53 smithi038 ceph-mon[96929]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 13991/238047 objects degraded (5.877%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:56:54.026 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:53 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 13991/238047 objects degraded (5.877%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:53 smithi055 ceph-mon[86906]: pgmap v367: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 13991/238047 objects degraded (5.877%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:56:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:53 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 13991/238047 objects degraded (5.877%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:55.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:54 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1909919891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:55.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:54 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1909919891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:55.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:54 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1909919891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:56.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:55 smithi055 ceph-mon[86906]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 13819/227406 objects degraded (6.077%); 2.7 MiB/s, 43 objects/s recovering 2024-04-26T04:56:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:55 smithi038 ceph-mon[93172]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 13819/227406 objects degraded (6.077%); 2.7 MiB/s, 43 objects/s recovering 2024-04-26T04:56:56.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:55 smithi038 ceph-mon[96929]: pgmap v368: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 13819/227406 objects degraded (6.077%); 2.7 MiB/s, 43 objects/s recovering 2024-04-26T04:56:57.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:56 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/456751831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:56 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/456751831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:56:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:56:56] "GET /metrics HTTP/1.1" 200 36260 "" "Prometheus/2.43.0" 2024-04-26T04:56:57.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:56 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/456751831' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:56:58.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:57 smithi055 ceph-mon[86906]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 13819/227406 objects degraded (6.077%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:56:58.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:57 smithi038 ceph-mon[93172]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 13819/227406 objects degraded (6.077%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:56:58.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:57 smithi038 ceph-mon[96929]: pgmap v369: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 13819/227406 objects degraded (6.077%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:56:59.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:58 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 13819/227406 objects degraded (6.077%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:59.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:58 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 13819/227406 objects degraded (6.077%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:56:59.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:58 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 13819/227406 objects degraded (6.077%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:00.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:59 smithi055 ceph-mon[86906]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13819/221763 objects degraded (6.231%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:57:00.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:56:59 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/290669149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:59 smithi038 ceph-mon[93172]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13819/221763 objects degraded (6.231%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:57:00.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:56:59 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/290669149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:00.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:59 smithi038 ceph-mon[96929]: pgmap v370: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13819/221763 objects degraded (6.231%); 1.8 MiB/s, 28 objects/s recovering 2024-04-26T04:57:00.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:56:59 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/290669149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:02.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:01 smithi055 ceph-mon[86906]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13630/216129 objects degraded (6.306%); 2.8 MiB/s, 44 objects/s recovering 2024-04-26T04:57:02.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:01 smithi038 ceph-mon[93172]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13630/216129 objects degraded (6.306%); 2.8 MiB/s, 44 objects/s recovering 2024-04-26T04:57:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:02.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:01 smithi038 ceph-mon[96929]: pgmap v371: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 13630/216129 objects degraded (6.306%); 2.8 MiB/s, 44 objects/s recovering 2024-04-26T04:57:02.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:03.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2626106457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:03.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2626106457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:03.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2626106457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:03 smithi038 ceph-mon[93172]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 13630/215448 objects degraded (6.326%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:57:04.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:03 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 13630/215448 objects degraded (6.326%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:03 smithi038 ceph-mon[96929]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 13630/215448 objects degraded (6.326%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:57:04.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:03 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 13630/215448 objects degraded (6.326%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:03 smithi055 ceph-mon[86906]: pgmap v372: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 13630/215448 objects degraded (6.326%); 1.9 MiB/s, 30 objects/s recovering 2024-04-26T04:57:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:03 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 13630/215448 objects degraded (6.326%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:05.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:04 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1121499989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:05.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:04 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:05.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:04 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1121499989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:05.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:04 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:05.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:04 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1121499989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:05.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:04 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:06.172 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:05 smithi055 ceph-mon[86906]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 13400/204588 objects degraded (6.550%); 2.5 MiB/s, 49 objects/s recovering 2024-04-26T04:57:06.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:05 smithi038 ceph-mon[93172]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 13400/204588 objects degraded (6.550%); 2.5 MiB/s, 49 objects/s recovering 2024-04-26T04:57:06.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:05 smithi038 ceph-mon[96929]: pgmap v373: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 13400/204588 objects degraded (6.550%); 2.5 MiB/s, 49 objects/s recovering 2024-04-26T04:57:07.225 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:57:06] "GET /metrics HTTP/1.1" 200 36263 "" "Prometheus/2.43.0" 2024-04-26T04:57:08.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:07 smithi038 ceph-mon[93172]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 13400/204588 objects degraded (6.550%); 1.6 MiB/s, 34 objects/s recovering 2024-04-26T04:57:08.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1554642679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:08.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:07 smithi038 ceph-mon[96929]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 13400/204588 objects degraded (6.550%); 1.6 MiB/s, 34 objects/s recovering 2024-04-26T04:57:08.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1554642679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:08.246 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:07 smithi055 ceph-mon[86906]: pgmap v374: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 13400/204588 objects degraded (6.550%); 1.6 MiB/s, 34 objects/s recovering 2024-04-26T04:57:08.246 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1554642679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:09.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:08 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 13400/204588 objects degraded (6.550%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:09.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:08 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 13400/204588 objects degraded (6.550%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:09.258 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:08 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 13400/204588 objects degraded (6.550%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[93172]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 13400/199257 objects degraded (6.725%); 1.6 MiB/s, 34 objects/s recovering 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[93172]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2548612201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[96929]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 13400/199257 objects degraded (6.725%); 1.6 MiB/s, 34 objects/s recovering 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[96929]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T04:57:10.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2548612201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:10.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:10.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:09 smithi055 ceph-mon[86906]: pgmap v375: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 13400/199257 objects degraded (6.725%); 1.6 MiB/s, 34 objects/s recovering 2024-04-26T04:57:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:09 smithi055 ceph-mon[86906]: osdmap e118: 8 total, 8 up, 8 in 2024-04-26T04:57:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:09 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2548612201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:11.124 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:10 smithi038 ceph-mon[93172]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T04:57:11.124 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:10 smithi038 ceph-mon[96929]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T04:57:11.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:10 smithi055 ceph-mon[86906]: osdmap e119: 8 total, 8 up, 8 in 2024-04-26T04:57:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:11 smithi038 ceph-mon[93172]: pgmap v378: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 13130/194247 objects degraded (6.759%); 1016 KiB/s, 62 objects/s recovering 2024-04-26T04:57:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:11 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:12.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:11 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:11 smithi038 ceph-mon[96929]: pgmap v378: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 13130/194247 objects degraded (6.759%); 1016 KiB/s, 62 objects/s recovering 2024-04-26T04:57:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:11 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:12.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:11 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:11 smithi055 ceph-mon[86906]: pgmap v378: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 13130/194247 objects degraded (6.759%); 1016 KiB/s, 62 objects/s recovering 2024-04-26T04:57:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:11 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:12.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:11 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:13.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:57:12.806+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3449541949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3449541949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.226 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3449541949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.257 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:57:13.258 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:14.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[96929]: pgmap v379: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 13130/193635 objects degraded (6.781%); 0 B/s, 33 objects/s recovering 2024-04-26T04:57:14.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:14.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:14.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 13130/193635 objects degraded (6.781%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:14.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[93172]: pgmap v379: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 13130/193635 objects degraded (6.781%); 0 B/s, 33 objects/s recovering 2024-04-26T04:57:14.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:14.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:14.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:13 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 13130/193635 objects degraded (6.781%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:14.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:13 smithi055 ceph-mon[86906]: pgmap v379: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 13130/193635 objects degraded (6.781%); 0 B/s, 33 objects/s recovering 2024-04-26T04:57:14.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:13 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:14.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:13 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:14.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:13 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 13130/193635 objects degraded (6.781%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:15.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:14 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/706403525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:15.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:14 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/706403525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:15.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:14 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/706403525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:16.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:15 smithi055 ceph-mon[86906]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 13130/187230 objects degraded (7.013%); 0 B/s, 33 objects/s recovering 2024-04-26T04:57:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:15 smithi038 ceph-mon[96929]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 13130/187230 objects degraded (7.013%); 0 B/s, 33 objects/s recovering 2024-04-26T04:57:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:15 smithi038 ceph-mon[93172]: pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s; 13130/187230 objects degraded (7.013%); 0 B/s, 33 objects/s recovering 2024-04-26T04:57:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:57:16] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.43.0" 2024-04-26T04:57:17.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:16 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:17.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:16 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:17.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:16 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:17 smithi055 ceph-mon[86906]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 12918/184725 objects degraded (6.993%); 0 B/s, 60 objects/s recovering 2024-04-26T04:57:18.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:17 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3932509591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:18 smithi038 ceph-mon[96929]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 12918/184725 objects degraded (6.993%); 0 B/s, 60 objects/s recovering 2024-04-26T04:57:18.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:18 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3932509591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:17 smithi038 ceph-mon[93172]: pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 12918/184725 objects degraded (6.993%); 0 B/s, 60 objects/s recovering 2024-04-26T04:57:18.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:17 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3932509591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:19.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:18 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 12918/184725 objects degraded (6.993%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 12918/184725 objects degraded (6.993%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:18 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 12918/184725 objects degraded (6.993%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:20.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:20 smithi055 ceph-mon[86906]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 12918/179217 objects degraded (7.208%); 0 B/s, 49 objects/s recovering 2024-04-26T04:57:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:20 smithi038 ceph-mon[96929]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 12918/179217 objects degraded (7.208%); 0 B/s, 49 objects/s recovering 2024-04-26T04:57:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:20 smithi038 ceph-mon[93172]: pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 12918/179217 objects degraded (7.208%); 0 B/s, 49 objects/s recovering 2024-04-26T04:57:21.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:21 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/36480545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:21.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:21 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/36480545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:21.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:21 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/36480545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:22.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:22 smithi055 ceph-mon[86906]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12757/173661 objects degraded (7.346%); 0 B/s, 34 objects/s recovering 2024-04-26T04:57:22.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:22 smithi038 ceph-mon[96929]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12757/173661 objects degraded (7.346%); 0 B/s, 34 objects/s recovering 2024-04-26T04:57:22.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:22 smithi038 ceph-mon[93172]: pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12757/173661 objects degraded (7.346%); 0 B/s, 34 objects/s recovering 2024-04-26T04:57:23.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:23 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3146077979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:23.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:23 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3146077979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:23 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3146077979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:24 smithi055 ceph-mon[86906]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 12757/172977 objects degraded (7.375%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:24.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 12757/172977 objects degraded (7.375%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:24.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:24 smithi038 ceph-mon[93172]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 12757/172977 objects degraded (7.375%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:24.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 12757/172977 objects degraded (7.375%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:24.375 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:24 smithi038 ceph-mon[96929]: pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 12757/172977 objects degraded (7.375%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:24.375 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 12757/172977 objects degraded (7.375%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:26.157 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:26 smithi038 ceph-mon[96929]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 12757/165291 objects degraded (7.718%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:26.157 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1675576347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:26.157 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:26 smithi038 ceph-mon[93172]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 12757/165291 objects degraded (7.718%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:26.157 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1675576347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:26.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:26 smithi055 ceph-mon[86906]: pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 12757/165291 objects degraded (7.718%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:26.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1675576347' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:57:26] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.43.0" 2024-04-26T04:57:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:28 smithi038 ceph-mon[93172]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 12589/162399 objects degraded (7.752%); 0 B/s, 45 objects/s recovering 2024-04-26T04:57:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:28 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:28.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:28 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3284613126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:28.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:27 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:57:27.823+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:28 smithi038 ceph-mon[96929]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 12589/162399 objects degraded (7.752%); 0 B/s, 45 objects/s recovering 2024-04-26T04:57:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:28 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:28.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:28 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3284613126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:28 smithi055 ceph-mon[86906]: pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 12589/162399 objects degraded (7.752%); 0 B/s, 45 objects/s recovering 2024-04-26T04:57:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:28 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:28.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:28 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3284613126' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:29 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:29 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:29.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:29 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 12589/162399 objects degraded (7.752%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:29 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:29 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:29 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 12589/162399 objects degraded (7.752%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:29 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:29 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:29 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 12589/162399 objects degraded (7.752%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:30.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:30 smithi055 ceph-mon[86906]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 12589/157194 objects degraded (8.009%); 0 B/s, 27 objects/s recovering 2024-04-26T04:57:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:30 smithi038 ceph-mon[93172]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 12589/157194 objects degraded (8.009%); 0 B/s, 27 objects/s recovering 2024-04-26T04:57:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:30 smithi038 ceph-mon[96929]: pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 12589/157194 objects degraded (8.009%); 0 B/s, 27 objects/s recovering 2024-04-26T04:57:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1049122291' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:31.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1049122291' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:31.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1049122291' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:32 smithi055 ceph-mon[86906]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12417/151869 objects degraded (8.176%); 0 B/s, 41 objects/s recovering 2024-04-26T04:57:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:32 smithi038 ceph-mon[93172]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12417/151869 objects degraded (8.176%); 0 B/s, 41 objects/s recovering 2024-04-26T04:57:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:32 smithi038 ceph-mon[96929]: pgmap v388: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12417/151869 objects degraded (8.176%); 0 B/s, 41 objects/s recovering 2024-04-26T04:57:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:34 smithi038 ceph-mon[93172]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 12417/151065 objects degraded (8.220%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:34 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/697879299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:34 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 12417/151065 objects degraded (8.220%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:34 smithi038 ceph-mon[96929]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 12417/151065 objects degraded (8.220%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:34 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/697879299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:34 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 12417/151065 objects degraded (8.220%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:34 smithi055 ceph-mon[86906]: pgmap v389: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 12417/151065 objects degraded (8.220%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:34 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/697879299' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:34 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 12417/151065 objects degraded (8.220%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:36 smithi038 ceph-mon[93172]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 12417/143394 objects degraded (8.659%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/574732273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:36 smithi038 ceph-mon[96929]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 12417/143394 objects degraded (8.659%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/574732273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:36 smithi055 ceph-mon[86906]: pgmap v390: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 12417/143394 objects degraded (8.659%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/574732273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:57:36] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.43.0" 2024-04-26T04:57:38.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:38 smithi038 ceph-mon[96929]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 12249/140475 objects degraded (8.720%); 0 B/s, 42 objects/s recovering 2024-04-26T04:57:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:38 smithi038 ceph-mon[93172]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 12249/140475 objects degraded (8.720%); 0 B/s, 42 objects/s recovering 2024-04-26T04:57:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:38 smithi055 ceph-mon[86906]: pgmap v391: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 12249/140475 objects degraded (8.720%); 0 B/s, 42 objects/s recovering 2024-04-26T04:57:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:39 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4195679512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:39 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 12249/140475 objects degraded (8.720%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:39.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:39 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4195679512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:39.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:39 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 12249/140475 objects degraded (8.720%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:39 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4195679512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:39 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 12249/140475 objects degraded (8.720%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:40 smithi038 ceph-mon[93172]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 12249/134925 objects degraded (9.078%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:40 smithi038 ceph-mon[96929]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 12249/134925 objects degraded (9.078%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:40 smithi055 ceph-mon[86906]: pgmap v392: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 12249/134925 objects degraded (9.078%); 0 B/s, 28 objects/s recovering 2024-04-26T04:57:41.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1090335053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:41.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1090335053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1090335053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:42 smithi038 ceph-mon[93172]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 12056/129417 objects degraded (9.316%); 0 B/s, 44 objects/s recovering 2024-04-26T04:57:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:42 smithi038 ceph-mon[96929]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 12056/129417 objects degraded (9.316%); 0 B/s, 44 objects/s recovering 2024-04-26T04:57:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:42 smithi055 ceph-mon[86906]: pgmap v393: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 12056/129417 objects degraded (9.316%); 0 B/s, 44 objects/s recovering 2024-04-26T04:57:43.189 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:42 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:57:42.840+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:43.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:43 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:43.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:43 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:43.507 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:43 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[93172]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12056/128730 objects degraded (9.365%); 0 B/s, 30 objects/s recovering 2024-04-26T04:57:44.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 12056/128730 objects degraded (9.365%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3370442855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[96929]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12056/128730 objects degraded (9.365%); 0 B/s, 30 objects/s recovering 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 12056/128730 objects degraded (9.365%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:44.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3370442855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:44 smithi055 ceph-mon[86906]: pgmap v394: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12056/128730 objects degraded (9.365%); 0 B/s, 30 objects/s recovering 2024-04-26T04:57:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:44 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:44 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 12056/128730 objects degraded (9.365%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3370442855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:46.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:46 smithi038 ceph-mon[93172]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 12056/120975 objects degraded (9.966%); 0 B/s, 30 objects/s recovering 2024-04-26T04:57:46.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:46 smithi038 ceph-mon[96929]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 12056/120975 objects degraded (9.966%); 0 B/s, 30 objects/s recovering 2024-04-26T04:57:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:46 smithi055 ceph-mon[86906]: pgmap v395: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 12056/120975 objects degraded (9.966%); 0 B/s, 30 objects/s recovering 2024-04-26T04:57:47.208 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:57:46] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.43.0" 2024-04-26T04:57:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:47 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/601343827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:47.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:47 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/601343827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:47.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:47 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/601343827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:57:48.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:48 smithi038 ceph-mon[93172]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 11865/118179 objects degraded (10.040%); 0 B/s, 45 objects/s recovering 2024-04-26T04:57:48.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:48 smithi038 ceph-mon[96929]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 11865/118179 objects degraded (10.040%); 0 B/s, 45 objects/s recovering 2024-04-26T04:57:48.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:48 smithi055 ceph-mon[86906]: pgmap v396: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 11865/118179 objects degraded (10.040%); 0 B/s, 45 objects/s recovering 2024-04-26T04:57:49.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:49 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 11865/118179 objects degraded (10.040%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:49.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:49 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/744178554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:49 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 11865/118179 objects degraded (10.040%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:49 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/744178554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:49.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:49 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 11865/118179 objects degraded (10.040%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:49.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:49 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/744178554' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:50 smithi055 ceph-mon[86906]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 11865/112449 objects degraded (10.551%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:50 smithi038 ceph-mon[93172]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 11865/112449 objects degraded (10.551%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:50.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:50 smithi038 ceph-mon[96929]: pgmap v397: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 11865/112449 objects degraded (10.551%); 0 B/s, 31 objects/s recovering 2024-04-26T04:57:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:52 smithi055 ceph-mon[86906]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 11661/106854 objects degraded (10.913%); 0 B/s, 48 objects/s recovering 2024-04-26T04:57:52.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:52 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2854541862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:52 smithi038 ceph-mon[96929]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 11661/106854 objects degraded (10.913%); 0 B/s, 48 objects/s recovering 2024-04-26T04:57:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:52 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2854541862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:52 smithi038 ceph-mon[93172]: pgmap v398: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 11661/106854 objects degraded (10.913%); 0 B/s, 48 objects/s recovering 2024-04-26T04:57:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:52 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2854541862' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:54.526 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:54 smithi038 ceph-mon[93172]: pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 11661/106173 objects degraded (10.983%); 0 B/s, 32 objects/s recovering 2024-04-26T04:57:54.526 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:54 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 11661/106173 objects degraded (10.983%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:54.526 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:54 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2356554123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:54.527 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:54 smithi038 ceph-mon[96929]: pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 11661/106173 objects degraded (10.983%); 0 B/s, 32 objects/s recovering 2024-04-26T04:57:54.527 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:54 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 11661/106173 objects degraded (10.983%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:54.527 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:54 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2356554123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:54 smithi055 ceph-mon[86906]: pgmap v399: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 11661/106173 objects degraded (10.983%); 0 B/s, 32 objects/s recovering 2024-04-26T04:57:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:54 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 11661/106173 objects degraded (10.983%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:54 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2356554123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:56 smithi055 ceph-mon[86906]: pgmap v400: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 11661/98505 objects degraded (11.838%); 0 B/s, 32 objects/s recovering 2024-04-26T04:57:56.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:56 smithi038 ceph-mon[93172]: pgmap v400: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 11661/98505 objects degraded (11.838%); 0 B/s, 32 objects/s recovering 2024-04-26T04:57:56.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:56 smithi038 ceph-mon[96929]: pgmap v400: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 11661/98505 objects degraded (11.838%); 0 B/s, 32 objects/s recovering 2024-04-26T04:57:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:57:56] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.43.0" 2024-04-26T04:57:57.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:57 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4195529179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:57.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:57 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4195529179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:57.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:57 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4195529179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:57:58.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:57:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:57:57.856+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:58 smithi055 ceph-mon[86906]: pgmap v401: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 11455/95655 objects degraded (11.975%); 0 B/s, 50 objects/s recovering 2024-04-26T04:57:58.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:58.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:58 smithi038 ceph-mon[93172]: pgmap v401: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 11455/95655 objects degraded (11.975%); 0 B/s, 50 objects/s recovering 2024-04-26T04:57:58.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:58.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:58 smithi038 ceph-mon[96929]: pgmap v401: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 11455/95655 objects degraded (11.975%); 0 B/s, 50 objects/s recovering 2024-04-26T04:57:58.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:59 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:59 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:57:59 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 11455/95655 objects degraded (11.975%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:59.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:59 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:59.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:59 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:59.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:57:59 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 11455/95655 objects degraded (11.975%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:57:59.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:59 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:57:59.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:59 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-26T04:57:59.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:57:59 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 11455/95655 objects degraded (11.975%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:00 smithi055 ceph-mon[86906]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 11455/89769 objects degraded (12.761%); 0 B/s, 34 objects/s recovering 2024-04-26T04:58:00.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:00 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3806106196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:00.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:00 smithi038 ceph-mon[93172]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 11455/89769 objects degraded (12.761%); 0 B/s, 34 objects/s recovering 2024-04-26T04:58:00.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3806106196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:00.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:00 smithi038 ceph-mon[96929]: pgmap v402: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.8 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 11455/89769 objects degraded (12.761%); 0 B/s, 34 objects/s recovering 2024-04-26T04:58:00.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3806106196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:02 smithi055 ceph-mon[86906]: pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11234/84294 objects degraded (13.327%); 0 B/s, 52 objects/s recovering 2024-04-26T04:58:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2654120006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:02.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:02 smithi038 ceph-mon[93172]: pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11234/84294 objects degraded (13.327%); 0 B/s, 52 objects/s recovering 2024-04-26T04:58:02.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:02.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2654120006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:02 smithi038 ceph-mon[96929]: pgmap v403: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11234/84294 objects degraded (13.327%); 0 B/s, 52 objects/s recovering 2024-04-26T04:58:02.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:02.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2654120006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:04 smithi055 ceph-mon[86906]: pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 11234/83625 objects degraded (13.434%); 0 B/s, 35 objects/s recovering 2024-04-26T04:58:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:04 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 11234/83625 objects degraded (13.434%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:04.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:04 smithi038 ceph-mon[93172]: pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 11234/83625 objects degraded (13.434%); 0 B/s, 35 objects/s recovering 2024-04-26T04:58:04.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:04 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 11234/83625 objects degraded (13.434%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:04.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:04 smithi038 ceph-mon[96929]: pgmap v404: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 11234/83625 objects degraded (13.434%); 0 B/s, 35 objects/s recovering 2024-04-26T04:58:04.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:04 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 11234/83625 objects degraded (13.434%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:05 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/694590481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:05.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:05 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/694590481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:05.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:05 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/694590481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:06 smithi055 ceph-mon[86906]: pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 11234/78516 objects degraded (14.308%); 0 B/s, 35 objects/s recovering 2024-04-26T04:58:06.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:06 smithi038 ceph-mon[93172]: pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 11234/78516 objects degraded (14.308%); 0 B/s, 35 objects/s recovering 2024-04-26T04:58:06.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:06 smithi038 ceph-mon[96929]: pgmap v405: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 11234/78516 objects degraded (14.308%); 0 B/s, 35 objects/s recovering 2024-04-26T04:58:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:58:06] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.43.0" 2024-04-26T04:58:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4227867122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:07.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4227867122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:07.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4227867122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:08 smithi055 ceph-mon[86906]: pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 11000/72894 objects degraded (15.090%); 0 B/s, 55 objects/s recovering 2024-04-26T04:58:08.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:08 smithi038 ceph-mon[93172]: pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 11000/72894 objects degraded (15.090%); 0 B/s, 55 objects/s recovering 2024-04-26T04:58:08.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:08 smithi038 ceph-mon[96929]: pgmap v406: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 11000/72894 objects degraded (15.090%); 0 B/s, 55 objects/s recovering 2024-04-26T04:58:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 11000/72894 objects degraded (15.090%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 11000/72894 objects degraded (15.090%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:09.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 11000/72894 objects degraded (15.090%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:10 smithi038 ceph-mon[93172]: pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11000/67515 objects degraded (16.293%); 0 B/s, 37 objects/s recovering 2024-04-26T04:58:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:10 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3121243747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:10 smithi038 ceph-mon[96929]: pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11000/67515 objects degraded (16.293%); 0 B/s, 37 objects/s recovering 2024-04-26T04:58:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:10 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3121243747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:10.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:10 smithi055 ceph-mon[86906]: pgmap v407: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11000/67515 objects degraded (16.293%); 0 B/s, 37 objects/s recovering 2024-04-26T04:58:10.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:10 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3121243747' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:12 smithi038 ceph-mon[93172]: pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 10752/61644 objects degraded (17.442%); 0 B/s, 58 objects/s recovering 2024-04-26T04:58:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:12 smithi038 ceph-mon[96929]: pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 10752/61644 objects degraded (17.442%); 0 B/s, 58 objects/s recovering 2024-04-26T04:58:12.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:12 smithi055 ceph-mon[86906]: pgmap v408: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 10752/61644 objects degraded (17.442%); 0 B/s, 58 objects/s recovering 2024-04-26T04:58:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:13 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2268826532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:13 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:13.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:13 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2268826532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:13.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:13 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:13.757 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:13 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2268826532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:13.757 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:13 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:14 smithi038 ceph-mon[96929]: pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 10752/60924 objects degraded (17.648%); 0 B/s, 40 objects/s recovering 2024-04-26T04:58:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:14 smithi038 ceph-mon[96929]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T04:58:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 10752/60924 objects degraded (17.648%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:14 smithi038 ceph-mon[93172]: pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 10752/60924 objects degraded (17.648%); 0 B/s, 40 objects/s recovering 2024-04-26T04:58:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:14 smithi038 ceph-mon[93172]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T04:58:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 10752/60924 objects degraded (17.648%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:14 smithi055 ceph-mon[86906]: pgmap v409: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 10752/60924 objects degraded (17.648%); 0 B/s, 40 objects/s recovering 2024-04-26T04:58:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:14 smithi055 ceph-mon[86906]: osdmap e120: 8 total, 8 up, 8 in 2024-04-26T04:58:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 10752/60924 objects degraded (17.648%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:15 smithi038 ceph-mon[93172]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T04:58:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:15 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3663757225' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:15.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:15 smithi038 ceph-mon[96929]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T04:58:15.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:15 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3663757225' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:15.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:15 smithi055 ceph-mon[86906]: osdmap e121: 8 total, 8 up, 8 in 2024-04-26T04:58:15.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:15 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3663757225' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:16 smithi055 ceph-mon[86906]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 10740/55995 objects degraded (19.180%); 0 B/s, 32 objects/s recovering 2024-04-26T04:58:16.681 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:16 smithi038 ceph-mon[93172]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 10740/55995 objects degraded (19.180%); 0 B/s, 32 objects/s recovering 2024-04-26T04:58:16.682 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:16 smithi038 ceph-mon[96929]: pgmap v412: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 10740/55995 objects degraded (19.180%); 0 B/s, 32 objects/s recovering 2024-04-26T04:58:16.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:58:16] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.43.0" 2024-04-26T04:58:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[93172]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10740/52116 objects degraded (20.608%); 0 B/s, 32 objects/s recovering 2024-04-26T04:58:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3909090515' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[96929]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10740/52116 objects degraded (20.608%); 0 B/s, 32 objects/s recovering 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3909090515' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:18 smithi055 ceph-mon[86906]: pgmap v413: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 84 active+clean; 1.1 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 10740/52116 objects degraded (20.608%); 0 B/s, 32 objects/s recovering 2024-04-26T04:58:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:18 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3909090515' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:18.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 10740/52116 objects degraded (20.608%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:58:19.271+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:19.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 10740/52116 objects degraded (20.608%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.726 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:19 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 10740/52116 objects degraded (20.608%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:58:19.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:19 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[93172]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 55 objects/s recovering 2024-04-26T04:58:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[93172]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 68 objects/s recovering 2024-04-26T04:58:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[93172]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 75 objects/s recovering 2024-04-26T04:58:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2019514340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:20.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[96929]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 55 objects/s recovering 2024-04-26T04:58:20.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[96929]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 68 objects/s recovering 2024-04-26T04:58:20.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[96929]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 75 objects/s recovering 2024-04-26T04:58:20.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:20.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:20.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:20 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2019514340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:20 smithi055 ceph-mon[86906]: pgmap v414: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 55 objects/s recovering 2024-04-26T04:58:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:20 smithi055 ceph-mon[86906]: pgmap v415: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 68 objects/s recovering 2024-04-26T04:58:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:20 smithi055 ceph-mon[86906]: pgmap v416: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 977 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 10311/46953 objects degraded (21.960%); 0 B/s, 75 objects/s recovering 2024-04-26T04:58:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:20 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:20 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:20 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2019514340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:21.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:21 smithi055 ceph-mon[86906]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 881 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 10311/42318 objects degraded (24.366%); 0 B/s, 64 objects/s recovering 2024-04-26T04:58:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:21 smithi038 ceph-mon[93172]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 881 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 10311/42318 objects degraded (24.366%); 0 B/s, 64 objects/s recovering 2024-04-26T04:58:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:21 smithi038 ceph-mon[96929]: pgmap v417: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 881 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 10311/42318 objects degraded (24.366%); 0 B/s, 64 objects/s recovering 2024-04-26T04:58:23.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:23 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1056237942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:23.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:23 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1056237942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:23 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1056237942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:24.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:24 smithi038 ceph-mon[93172]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 747 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 10140/35922 objects degraded (28.228%); 0 B/s, 71 objects/s recovering 2024-04-26T04:58:24.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:24 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 10140/35922 objects degraded (28.228%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:24.375 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:24 smithi038 ceph-mon[96929]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 747 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 10140/35922 objects degraded (28.228%); 0 B/s, 71 objects/s recovering 2024-04-26T04:58:24.376 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:24 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 10140/35922 objects degraded (28.228%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:24 smithi055 ceph-mon[86906]: pgmap v418: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 747 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 10140/35922 objects degraded (28.228%); 0 B/s, 71 objects/s recovering 2024-04-26T04:58:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:24 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 10140/35922 objects degraded (28.228%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:26 smithi038 ceph-mon[93172]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 643 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 10140/30900 objects degraded (32.816%); 0 B/s, 71 objects/s recovering 2024-04-26T04:58:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2007036309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:26.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:26 smithi038 ceph-mon[96929]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 643 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 10140/30900 objects degraded (32.816%); 0 B/s, 71 objects/s recovering 2024-04-26T04:58:26.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2007036309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:26 smithi055 ceph-mon[86906]: pgmap v419: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 643 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 10140/30900 objects degraded (32.816%); 0 B/s, 71 objects/s recovering 2024-04-26T04:58:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2007036309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:58:26] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.43.0" 2024-04-26T04:58:28.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:28 smithi038 ceph-mon[93172]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 643 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 10140/30900 objects degraded (32.816%); 0 B/s, 20 objects/s recovering 2024-04-26T04:58:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:28 smithi038 ceph-mon[96929]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 643 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 10140/30900 objects degraded (32.816%); 0 B/s, 20 objects/s recovering 2024-04-26T04:58:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:28 smithi055 ceph-mon[86906]: pgmap v420: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 643 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 10140/30900 objects degraded (32.816%); 0 B/s, 20 objects/s recovering 2024-04-26T04:58:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:29 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3481645446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:29 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 10140/30900 objects degraded (32.816%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:29 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3481645446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:29 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 10140/30900 objects degraded (32.816%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:29 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3481645446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:29 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 10140/30900 objects degraded (32.816%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:30.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:30 smithi038 ceph-mon[93172]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 513 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 9968/24684 objects degraded (40.382%); 0 B/s, 34 objects/s recovering 2024-04-26T04:58:30.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:30 smithi038 ceph-mon[96929]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 513 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 9968/24684 objects degraded (40.382%); 0 B/s, 34 objects/s recovering 2024-04-26T04:58:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:30 smithi055 ceph-mon[86906]: pgmap v421: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 513 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 9968/24684 objects degraded (40.382%); 0 B/s, 34 objects/s recovering 2024-04-26T04:58:31.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1787762531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:31.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1787762531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:31.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1787762531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:32 smithi038 ceph-mon[93172]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 411 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9968/19791 objects degraded (50.366%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:32 smithi038 ceph-mon[96929]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 411 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9968/19791 objects degraded (50.366%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:32.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:32 smithi055 ceph-mon[86906]: pgmap v422: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 411 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 9968/19791 objects degraded (50.366%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-mon[93172]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 277 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 9792/13341 objects degraded (73.398%); 0 B/s, 43 objects/s recovering 2024-04-26T04:58:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1755837527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 9792/13341 objects degraded (73.398%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:34.475 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:58:34.288+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-mon[96929]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 277 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 9792/13341 objects degraded (73.398%); 0 B/s, 43 objects/s recovering 2024-04-26T04:58:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1755837527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:34.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:34 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 9792/13341 objects degraded (73.398%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:34 smithi055 ceph-mon[86906]: pgmap v423: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 277 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 9792/13341 objects degraded (73.398%); 0 B/s, 43 objects/s recovering 2024-04-26T04:58:34.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:34 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1755837527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:34.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:34 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 9792/13341 objects degraded (73.398%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:35 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:35 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:35.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:35 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:35.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:35 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:35.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:35 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:35.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:35 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:35 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:35 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:35 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-26T04:58:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:36 smithi038 ceph-mon[93172]: pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 167 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 9792/8085 objects degraded (121.113%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3626186328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:36 smithi038 ceph-mon[96929]: pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 167 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 9792/8085 objects degraded (121.113%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:36.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3626186328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:36 smithi055 ceph-mon[86906]: pgmap v424: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 167 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 9792/8085 objects degraded (121.113%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3626186328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:37.213 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Removed 111331 objects 2024-04-26T04:58:37.213 INFO:tasks.radosbench.radosbench.0.smithi038.stdout:Clean up completed and total clean up time :148.418 2024-04-26T04:58:37.223 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:58:36] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.43.0" 2024-04-26T04:58:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:38 smithi038 ceph-mon[93172]: pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 167 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9792/8085 objects degraded (121.113%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:38 smithi038 ceph-mon[96929]: pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 167 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9792/8085 objects degraded (121.113%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:38 smithi055 ceph-mon[86906]: pgmap v425: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 167 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 9792/8085 objects degraded (121.113%); 0 B/s, 28 objects/s recovering 2024-04-26T04:58:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:39 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/13843998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:39.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:39 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 9792/8085 objects degraded (121.113%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:39.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:39 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/13843998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:39.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:39 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 9792/8085 objects degraded (121.113%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:39 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/13843998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:39 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 9792/8085 objects degraded (121.113%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:40 smithi038 ceph-mon[93172]: pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 54 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9466/2619 objects degraded (361.436%); 0 B/s, 56 objects/s recovering 2024-04-26T04:58:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:40 smithi038 ceph-mon[96929]: pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 54 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9466/2619 objects degraded (361.436%); 0 B/s, 56 objects/s recovering 2024-04-26T04:58:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:40 smithi055 ceph-mon[86906]: pgmap v426: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 54 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9466/2619 objects degraded (361.436%); 0 B/s, 56 objects/s recovering 2024-04-26T04:58:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3617207271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3617207271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3617207271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:41.780 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-04-26T04:58:41.780 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-04-26T04:58:42.084 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:58:42.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:42 smithi038 ceph-mon[93172]: pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9466/72 objects degraded (13147.222%); 0 B/s, 41 objects/s recovering 2024-04-26T04:58:42.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:42 smithi038 ceph-mon[96929]: pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9466/72 objects degraded (13147.222%); 0 B/s, 41 objects/s recovering 2024-04-26T04:58:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:42 smithi055 ceph-mon[86906]: pgmap v427: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9466/72 objects degraded (13147.222%); 0 B/s, 41 objects/s recovering 2024-04-26T04:58:44.260 INFO:teuthology.orchestra.run.smithi038.stderr:pool 'unique_pool_3' removed 2024-04-26T04:58:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:44 smithi055 ceph-mon[86906]: pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 8223/72 objects degraded (11420.833%); 0 B/s, 145 objects/s recovering 2024-04-26T04:58:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/627570042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 8223/72 objects degraded (11420.833%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:44.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2066562194' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[93172]: pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 8223/72 objects degraded (11420.833%); 0 B/s, 145 objects/s recovering 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/627570042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 8223/72 objects degraded (11420.833%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2066562194' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[96929]: pgmap v428: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 8223/72 objects degraded (11420.833%); 0 B/s, 145 objects/s recovering 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/627570042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 8223/72 objects degraded (11420.833%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-26T04:58:44.617 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2066562194' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-26T04:58:44.981 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-26T04:58:44.982 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-04-26T04:58:44.982 DEBUG:teuthology.parallel:result is None 2024-04-26T04:58:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:45 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2066562194' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:58:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:45 smithi055 ceph-mon[86906]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T04:58:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:45 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2066562194' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:58:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:45 smithi038 ceph-mon[93172]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T04:58:45.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:45 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2066562194' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-26T04:58:45.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:45 smithi038 ceph-mon[96929]: osdmap e122: 8 total, 8 up, 8 in 2024-04-26T04:58:46.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:46 smithi055 ceph-mon[86906]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:58:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:46 smithi055 ceph-mon[86906]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8223/72 objects degraded (11420.833%), 4 pgs degraded, 4 pgs undersized) 2024-04-26T04:58:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:46 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4082320113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-mon[93172]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:58:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-mon[93172]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8223/72 objects degraded (11420.833%), 4 pgs degraded, 4 pgs undersized) 2024-04-26T04:58:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4082320113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-mon[96929]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:58:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-mon[96929]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8223/72 objects degraded (11420.833%), 4 pgs degraded, 4 pgs undersized) 2024-04-26T04:58:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4082320113' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:58:46] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-26T04:58:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:48.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:47 smithi055 ceph-mon[86906]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:58:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:48.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:47 smithi038 ceph-mon[93172]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:58:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:58:48.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:47 smithi038 ceph-mon[96929]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:58:49.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:48 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/29658413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:48 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/29658413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:49.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:48 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/29658413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:49 smithi055 ceph-mon[86906]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:49 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:50.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:49 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:50.107 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:49 smithi055 ceph-mon[86906]: Upgrade: osd.2 is safe to restart 2024-04-26T04:58:50.165 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[93172]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:50.165 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:50.165 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:50.166 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[93172]: Upgrade: osd.2 is safe to restart 2024-04-26T04:58:50.166 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[96929]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:50.166 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:50.166 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-26T04:58:50.166 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:49 smithi038 ceph-mon[96929]: Upgrade: osd.2 is safe to restart 2024-04-26T04:58:51.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:50 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3448317007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:50 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3448317007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:51.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:50 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3448317007' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:52.163 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[93172]: Upgrade: Updating osd.2 2024-04-26T04:58:52.163 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:52.163 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:58:52.163 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[93172]: Deploying daemon osd.2 on smithi038 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[93172]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[96929]: Upgrade: Updating osd.2 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[96929]: Deploying daemon osd.2 on smithi038 2024-04-26T04:58:52.164 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:51 smithi038 ceph-mon[96929]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:51 smithi055 ceph-mon[86906]: Upgrade: Updating osd.2 2024-04-26T04:58:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:51 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:51 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-26T04:58:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:51 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:58:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:51 smithi055 ceph-mon[86906]: Deploying daemon osd.2 on smithi038 2024-04-26T04:58:52.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:51 smithi055 ceph-mon[86906]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:53.597 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:53 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:53.597 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:53 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1002079726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:53.597 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:53 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:53.597 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:53 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1002079726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:53 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:58:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:53 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1002079726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:54.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:54 smithi055 ceph-mon[86906]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:54.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:54 smithi038 ceph-mon[96929]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:54.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:54 smithi038 systemd[1]: Stopping Ceph osd.2 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:58:54.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:54 smithi038 ceph-mon[93172]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:55.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[52734]: 2024-04-26T04:58:54.825+0000 7fac59db4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:58:55.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[52734]: 2024-04-26T04:58:54.825+0000 7fac59db4700 -1 osd.2 122 *** Got signal Terminated *** 2024-04-26T04:58:55.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[52734]: 2024-04-26T04:58:54.825+0000 7fac59db4700 -1 osd.2 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:58:55.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:55 smithi055 ceph-mon[86906]: osd.2 marked itself down and dead 2024-04-26T04:58:55.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:55 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3025357309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:55 smithi038 ceph-mon[93172]: osd.2 marked itself down and dead 2024-04-26T04:58:55.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:55 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3025357309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:55.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:55 smithi038 ceph-mon[96929]: osd.2 marked itself down and dead 2024-04-26T04:58:55.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:55 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3025357309' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:56.186 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:55 smithi038 podman[122666]: 2024-04-26 04:58:55.885944964 +0000 UTC m=+1.161671466 container died 05094c28a4cd0c8d3297d9e215c0aa150431d407300220993fe153452eef840f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:58:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-mon[93172]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-mon[93172]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:58:56.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-mon[93172]: osdmap e123: 8 total, 7 up, 8 in 2024-04-26T04:58:56.474 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:56 smithi038 podman[122666]: 2024-04-26 04:58:56.285010481 +0000 UTC m=+1.560736988 container cleanup 05094c28a4cd0c8d3297d9e215c0aa150431d407300220993fe153452eef840f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c) 2024-04-26T04:58:56.474 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:56 smithi038 bash[122666]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2 2024-04-26T04:58:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-mon[96929]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-mon[96929]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:58:56.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-mon[96929]: osdmap e123: 8 total, 7 up, 8 in 2024-04-26T04:58:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:56 smithi055 ceph-mon[86906]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:56 smithi055 ceph-mon[86906]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:58:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:56 smithi055 ceph-mon[86906]: osdmap e123: 8 total, 7 up, 8 in 2024-04-26T04:58:56.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:58:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:58:56] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-26T04:58:56.974 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:56 smithi038 podman[122710]: 2024-04-26 04:58:56.686439449 +0000 UTC m=+0.799470217 container remove 05094c28a4cd0c8d3297d9e215c0aa150431d407300220993fe153452eef840f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T04:58:57.382 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:57 smithi038 podman[122799]: 2024-04-26 04:58:56.97080165 +0000 UTC m=+0.018961461 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:57.382 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:57 smithi038 podman[122799]: 2024-04-26 04:58:57.070833608 +0000 UTC m=+0.118993405 container create b2724a45c511b6126b0eaefaf96396c1496810b73a73840d596077f554e4b104 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T04:58:57.382 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:57 smithi038 podman[122799]: 2024-04-26 04:58:57.298484246 +0000 UTC m=+0.346644043 container init b2724a45c511b6126b0eaefaf96396c1496810b73a73840d596077f554e4b104 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-26T04:58:57.382 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:57 smithi038 podman[122799]: 2024-04-26 04:58:57.302144353 +0000 UTC m=+0.350304158 container start b2724a45c511b6126b0eaefaf96396c1496810b73a73840d596077f554e4b104 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T04:58:57.382 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:57 smithi038 podman[122799]: 2024-04-26 04:58:57.381745761 +0000 UTC m=+0.429905570 container attach b2724a45c511b6126b0eaefaf96396c1496810b73a73840d596077f554e4b104 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-26T04:58:57.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:57 smithi038 ceph-mon[93172]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T04:58:57.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:57 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2952360820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:57 smithi038 ceph-mon[96929]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T04:58:57.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:57 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2952360820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:57.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:57 smithi038 podman[122799]: 2024-04-26 04:58:57.624238659 +0000 UTC m=+0.672398456 container died b2724a45c511b6126b0eaefaf96396c1496810b73a73840d596077f554e4b104 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-26T04:58:57.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:57 smithi055 ceph-mon[86906]: osdmap e124: 8 total, 7 up, 8 in 2024-04-26T04:58:57.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:57 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2952360820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:58:58.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:58 smithi038 ceph-mon[93172]: pgmap v438: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:58.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:58 smithi038 ceph-mon[96929]: pgmap v438: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:58.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 podman[122851]: 2024-04-26 04:58:58.286130736 +0000 UTC m=+0.655937453 container remove b2724a45c511b6126b0eaefaf96396c1496810b73a73840d596077f554e4b104 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T04:58:58.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-26T04:58:58.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 systemd[1]: Stopped Ceph osd.2 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:58:58.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.2.service: Consumed 27min 24.757s CPU time. 2024-04-26T04:58:58.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 systemd[1]: Starting Ceph osd.2 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:58:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:58 smithi055 ceph-mon[86906]: pgmap v438: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-04-26T04:58:59.002 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 podman[122957]: 2024-04-26 04:58:58.653403613 +0000 UTC m=+0.018513509 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:58:59.011 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 podman[122957]: 2024-04-26 04:58:58.763830655 +0000 UTC m=+0.128940552 container create ab64d84b41a668fa4b5ece870dcdcd4305930abcc7a0f981651334207e5e40dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:59.011 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:58 smithi038 podman[122957]: 2024-04-26 04:58:58.99853088 +0000 UTC m=+0.363640777 container init ab64d84b41a668fa4b5ece870dcdcd4305930abcc7a0f981651334207e5e40dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb) 2024-04-26T04:58:59.011 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 podman[122957]: 2024-04-26 04:58:59.001878181 +0000 UTC m=+0.366988084 container start ab64d84b41a668fa4b5ece870dcdcd4305930abcc7a0f981651334207e5e40dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T04:58:59.442 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 podman[122957]: 2024-04-26 04:58:59.098504705 +0000 UTC m=+0.463614609 container attach ab64d84b41a668fa4b5ece870dcdcd4305930abcc7a0f981651334207e5e40dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-26T04:58:59.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-mon[96929]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T04:58:59.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-mon[93172]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T04:58:59.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:58:59.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T04:58:59.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 bash[122957]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate[122972]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-26T04:58:59.725 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:58:59 smithi038 podman[122957]: 2024-04-26 04:58:59.534782491 +0000 UTC m=+0.899892383 container died ab64d84b41a668fa4b5ece870dcdcd4305930abcc7a0f981651334207e5e40dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:58:59.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:58:59 smithi055 ceph-mon[86906]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T04:59:00.487 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:00 smithi038 ceph-mon[93172]: pgmap v439: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 58 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:00.487 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/368960555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:00.487 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 podman[123112]: 2024-04-26 04:59:00.23589868 +0000 UTC m=+0.695180271 container remove ab64d84b41a668fa4b5ece870dcdcd4305930abcc7a0f981651334207e5e40dc (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:59:00.487 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 podman[123179]: 2024-04-26 04:59:00.366088199 +0000 UTC m=+0.019225563 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:00.488 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:00 smithi038 ceph-mon[96929]: pgmap v439: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 58 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:00.488 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/368960555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:00.762 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 podman[123179]: 2024-04-26 04:59:00.486811891 +0000 UTC m=+0.139949233 container create f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS) 2024-04-26T04:59:00.762 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 podman[123179]: 2024-04-26 04:59:00.689571439 +0000 UTC m=+0.342708778 container init f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-26T04:59:00.762 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 podman[123179]: 2024-04-26 04:59:00.692882763 +0000 UTC m=+0.346020111 container start f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-04-26T04:59:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:00 smithi055 ceph-mon[86906]: pgmap v439: 73 pgs: 1 active+undersized+degraded, 1 stale+active+clean, 13 active+undersized, 58 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:00 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/368960555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:01.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 ceph-osd[123198]: -- 172.21.15.38:0/3510102330 <== mon.0 v2:172.21.15.38:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5637e5789680 con 0x5637e5752c00 2024-04-26T04:59:01.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 bash[123179]: f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 2024-04-26T04:59:01.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:00 smithi038 systemd[1]: Started Ceph osd.2 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:59:01.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:01 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[123194]: 2024-04-26T04:59:01.278+0000 7fa11c5a37c0 -1 Falling back to public interface 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: pgmap v440: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/419799843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: pgmap v440: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:02.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/419799843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:02.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: pgmap v440: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:02.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/419799843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:04 smithi038 ceph-mon[93172]: pgmap v441: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:04 smithi038 ceph-mon[96929]: pgmap v441: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:04.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:04 smithi055 ceph-mon[86906]: pgmap v441: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:05 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2285655173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:05.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:05 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:05 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2285655173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:05.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:05 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:05 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2285655173' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:05.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:05 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:06 smithi038 ceph-mon[93172]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:06 smithi038 ceph-mon[96929]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:06 smithi055 ceph-mon[86906]: pgmap v442: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:06.823 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[123194]: 2024-04-26T04:59:06.554+0000 7fa11c5a37c0 -1 osd.2 122 log_to_monitors true 2024-04-26T04:59:07.134 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:59:06] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2024-04-26T04:59:07.474 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[123194]: 2024-04-26T04:59:07.194+0000 7fa10d9fb700 -1 osd.2 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:59:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[93172]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:59:07.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2306674419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[96929]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:59:07.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2306674419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:07 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:07 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:07 smithi055 ceph-mon[86906]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-26T04:59:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2306674419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[93172]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[93172]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[93172]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[93172]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[96929]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[96929]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:59:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[96929]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T04:59:08.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:08 smithi038 ceph-mon[96929]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:08 smithi055 ceph-mon[86906]: pgmap v443: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:08 smithi055 ceph-mon[86906]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-26T04:59:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:08 smithi055 ceph-mon[86906]: osdmap e125: 8 total, 7 up, 8 in 2024-04-26T04:59:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:08 smithi055 ceph-mon[86906]: from='osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599] boot 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2973123871' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:09.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[93172]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599] boot 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2973123871' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:09.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:09 smithi038 ceph-mon[96929]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T04:59:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:59:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: osd.2 [v2:172.21.15.38:6818/3191109599,v1:172.21.15.38:6819/3191109599] boot 2024-04-26T04:59:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: osdmap e126: 8 total, 8 up, 8 in 2024-04-26T04:59:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-26T04:59:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2973123871' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:09.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:09 smithi055 ceph-mon[86906]: osdmap e127: 8 total, 8 up, 8 in 2024-04-26T04:59:10.210 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:59:10.006+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-26T04:59:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: pgmap v446: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: pgmap v446: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:10.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.476 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:10 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: pgmap v446: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:10.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:10 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[93172]: pgmap v448: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[93172]: pgmap v449: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:11.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-26T04:59:11.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[96929]: pgmap v448: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:11.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[96929]: pgmap v449: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:11.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:11.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:11 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-26T04:59:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:11 smithi055 ceph-mon[86906]: pgmap v448: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:11 smithi055 ceph-mon[86906]: pgmap v449: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:11 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:11.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:11 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-04-26T04:59:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:12 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/168725483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:12 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/168725483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:12 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/168725483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:13.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:13 smithi055 ceph-mon[86906]: pgmap v450: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:13.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:13 smithi038 ceph-mon[93172]: pgmap v450: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:13.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:13 smithi038 ceph-mon[96929]: pgmap v450: 73 pgs: 1 active+undersized+degraded, 14 active+undersized, 58 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-26T04:59:14.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:14 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T04:59:14.508 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:14 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3482812904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:14 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T04:59:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:14 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3482812904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:14 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-26T04:59:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:14 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3482812904' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:15 smithi055 ceph-mon[86906]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:15.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:15 smithi055 ceph-mon[86906]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-26T04:59:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:15 smithi038 ceph-mon[93172]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:15 smithi038 ceph-mon[93172]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-26T04:59:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:15 smithi038 ceph-mon[96929]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:15 smithi038 ceph-mon[96929]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-26T04:59:16.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:16 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1994980819' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:16.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:16 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1994980819' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:16.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:16 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1994980819' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:59:16] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2024-04-26T04:59:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:17 smithi055 ceph-mon[86906]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:17 smithi038 ceph-mon[93172]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:17 smithi038 ceph-mon[96929]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:17.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:19 smithi055 ceph-mon[86906]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:19 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3000039410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:19 smithi038 ceph-mon[93172]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:19.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:19 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3000039410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:19 smithi038 ceph-mon[96929]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:19.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:19 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3000039410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:21 smithi055 ceph-mon[86906]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:21.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:21 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1132226281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:21.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:21 smithi038 ceph-mon[93172]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:21.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:21 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1132226281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:21.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:21 smithi038 ceph-mon[96929]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:21.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:21 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1132226281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:23.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:23 smithi055 ceph-mon[86906]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:23.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:23 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:23.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:23 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1509746903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:23 smithi038 ceph-mon[93172]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:23 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1509746903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:23.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:23 smithi038 ceph-mon[96929]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:23.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:23.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:23 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1509746903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:25.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:25 smithi038 ceph-mon[93172]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:25.644 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:25 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:25.644 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:25 smithi038 ceph-mon[96929]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:25.644 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:25 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:25.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:25 smithi055 ceph-mon[86906]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:25.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:25 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:26.682 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:26.682 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-mon[93172]: Upgrade: osd.3 is safe to restart 2024-04-26T04:59:26.682 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/441775398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:26.683 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:26.683 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-mon[96929]: Upgrade: osd.3 is safe to restart 2024-04-26T04:59:26.683 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/441775398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:26 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-26T04:59:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:26 smithi055 ceph-mon[86906]: Upgrade: osd.3 is safe to restart 2024-04-26T04:59:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/441775398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:26.973 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:59:26] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2024-04-26T04:59:27.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[96929]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:27.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:27.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:59:27.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:27.701 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[93172]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:27.702 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:27.702 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:59:27.702 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:27 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:27.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:27 smithi055 ceph-mon[86906]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:27.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:27.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-26T04:59:27.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:27 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:28.656 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:28 smithi038 ceph-mon[93172]: Upgrade: Updating osd.3 2024-04-26T04:59:28.656 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:28 smithi038 ceph-mon[93172]: Deploying daemon osd.3 on smithi038 2024-04-26T04:59:28.657 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:28 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4233650064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:28.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:28 smithi038 ceph-mon[96929]: Upgrade: Updating osd.3 2024-04-26T04:59:28.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:28 smithi038 ceph-mon[96929]: Deploying daemon osd.3 on smithi038 2024-04-26T04:59:28.657 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:28 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4233650064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:28 smithi055 ceph-mon[86906]: Upgrade: Updating osd.3 2024-04-26T04:59:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:28 smithi055 ceph-mon[86906]: Deploying daemon osd.3 on smithi038 2024-04-26T04:59:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:28 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4233650064' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:29.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:29 smithi038 ceph-mon[93172]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:29.748 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:29 smithi038 ceph-mon[96929]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:29 smithi055 ceph-mon[86906]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:30.494 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:30 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4130763339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:30.495 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:30 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4130763339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:30.495 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:30 smithi038 systemd[1]: Stopping Ceph osd.3 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:59:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:30 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4130763339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:30.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[58918]: 2024-04-26T04:59:30.493+0000 7ff84278a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T04:59:30.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[58918]: 2024-04-26T04:59:30.493+0000 7ff84278a700 -1 osd.3 127 *** Got signal Terminated *** 2024-04-26T04:59:30.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[58918]: 2024-04-26T04:59:30.493+0000 7ff84278a700 -1 osd.3 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T04:59:31.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:31 smithi038 ceph-mon[93172]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:31.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:31 smithi038 ceph-mon[93172]: osd.3 marked itself down and dead 2024-04-26T04:59:31.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:31 smithi038 ceph-mon[96929]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:31.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:31 smithi038 ceph-mon[96929]: osd.3 marked itself down and dead 2024-04-26T04:59:31.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:31 smithi055 ceph-mon[86906]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:31.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:31 smithi055 ceph-mon[86906]: osd.3 marked itself down and dead 2024-04-26T04:59:32.177 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:31 smithi038 podman[127876]: 2024-04-26 04:59:31.89317632 +0000 UTC m=+1.520807373 container died 61ce04224439fcbc9c597001536eb64d4409dc0f611c500463565e443020bfc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, RELEASE=quincy-c61472c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T04:59:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[93172]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:59:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[93172]: osdmap e128: 8 total, 7 up, 8 in 2024-04-26T04:59:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:32.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[96929]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:59:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[96929]: osdmap e128: 8 total, 7 up, 8 in 2024-04-26T04:59:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:32.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:32.475 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:32 smithi038 podman[127876]: 2024-04-26 04:59:32.368312453 +0000 UTC m=+1.995943496 container cleanup 61ce04224439fcbc9c597001536eb64d4409dc0f611c500463565e443020bfc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-04-26T04:59:32.475 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:32 smithi038 bash[127876]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3 2024-04-26T04:59:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:32 smithi055 ceph-mon[86906]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-26T04:59:32.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:32 smithi055 ceph-mon[86906]: osdmap e128: 8 total, 7 up, 8 in 2024-04-26T04:59:32.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:32.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:33.224 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:32 smithi038 podman[127893]: 2024-04-26 04:59:32.878810079 +0000 UTC m=+0.979728085 container remove 61ce04224439fcbc9c597001536eb64d4409dc0f611c500463565e443020bfc9 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, ceph=True, io.buildah.version=1.33.5) 2024-04-26T04:59:33.491 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:33 smithi038 ceph-mon[93172]: pgmap v461: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:33.492 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:33 smithi038 ceph-mon[93172]: osdmap e129: 8 total, 7 up, 8 in 2024-04-26T04:59:33.492 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:33 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/605738159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:33.492 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:33 smithi038 ceph-mon[96929]: pgmap v461: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:33.492 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:33 smithi038 ceph-mon[96929]: osdmap e129: 8 total, 7 up, 8 in 2024-04-26T04:59:33.492 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:33 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/605738159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:33.492 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:33 smithi038 podman[128007]: 2024-04-26 04:59:33.131540661 +0000 UTC m=+0.018564624 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:33.492 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:33 smithi038 podman[128007]: 2024-04-26 04:59:33.237995389 +0000 UTC m=+0.125019340 container create a2cc671e8723dec21bfd9b7a86cffbe8046d2df1699edaade628ced4e4b341e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, ceph=True) 2024-04-26T04:59:33.798 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:33 smithi038 podman[128007]: 2024-04-26 04:59:33.491359231 +0000 UTC m=+0.378383179 container init a2cc671e8723dec21bfd9b7a86cffbe8046d2df1699edaade628ced4e4b341e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T04:59:33.799 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:33 smithi038 podman[128007]: 2024-04-26 04:59:33.494788625 +0000 UTC m=+0.381812575 container start a2cc671e8723dec21bfd9b7a86cffbe8046d2df1699edaade628ced4e4b341e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:59:33.799 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:33 smithi038 podman[128007]: 2024-04-26 04:59:33.574451338 +0000 UTC m=+0.461475299 container attach a2cc671e8723dec21bfd9b7a86cffbe8046d2df1699edaade628ced4e4b341e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T04:59:33.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:33 smithi055 ceph-mon[86906]: pgmap v461: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-04-26T04:59:33.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:33 smithi055 ceph-mon[86906]: osdmap e129: 8 total, 7 up, 8 in 2024-04-26T04:59:33.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:33 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/605738159' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:34.224 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:33 smithi038 podman[128007]: 2024-04-26 04:59:33.798316988 +0000 UTC m=+0.685340936 container died a2cc671e8723dec21bfd9b7a86cffbe8046d2df1699edaade628ced4e4b341e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True) 2024-04-26T04:59:34.588 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:34 smithi038 ceph-mon[93172]: pgmap v463: 73 pgs: 2 active+undersized+degraded, 3 peering, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:59:34.588 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:34 smithi038 ceph-mon[93172]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:34.588 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:34 smithi038 ceph-mon[96929]: pgmap v463: 73 pgs: 2 active+undersized+degraded, 3 peering, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:59:34.588 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:34 smithi038 ceph-mon[96929]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:34 smithi055 ceph-mon[86906]: pgmap v463: 73 pgs: 2 active+undersized+degraded, 3 peering, 2 stale+active+clean, 19 active+undersized, 47 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-26T04:59:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:34 smithi055 ceph-mon[86906]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:34.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:34 smithi038 podman[128033]: 2024-04-26 04:59:34.587066806 +0000 UTC m=+0.782204934 container remove a2cc671e8723dec21bfd9b7a86cffbe8046d2df1699edaade628ced4e4b341e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-26T04:59:34.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:34 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-26T04:59:34.975 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:34 smithi038 systemd[1]: Stopped Ceph osd.3 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:59:34.975 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:34 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.3.service: Consumed 30min 8.445s CPU time. 2024-04-26T04:59:34.975 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:34 smithi038 systemd[1]: Starting Ceph osd.3 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T04:59:35.375 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 podman[128164]: 2024-04-26 04:59:34.957701778 +0000 UTC m=+0.019101525 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:35.375 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 podman[128164]: 2024-04-26 04:59:35.088931915 +0000 UTC m=+0.150331634 container create 20c115c1cbcc747e812b833795c96df6a5107e381ec08102f6fdf74d52c8d24d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-26T04:59:35.375 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 podman[128164]: 2024-04-26 04:59:35.283195065 +0000 UTC m=+0.344594793 container init 20c115c1cbcc747e812b833795c96df6a5107e381ec08102f6fdf74d52c8d24d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-26T04:59:35.375 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 podman[128164]: 2024-04-26 04:59:35.28710856 +0000 UTC m=+0.348508280 container start 20c115c1cbcc747e812b833795c96df6a5107e381ec08102f6fdf74d52c8d24d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-26T04:59:35.375 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 podman[128164]: 2024-04-26 04:59:35.374776431 +0000 UTC m=+0.436176159 container attach 20c115c1cbcc747e812b833795c96df6a5107e381ec08102f6fdf74d52c8d24d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-26T04:59:35.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/491919593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/491919593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:35.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:35 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/491919593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 bash[128164]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-26T04:59:36.063 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-26T04:59:36.064 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-26T04:59:36.064 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate[128178]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-26T04:59:36.064 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:35 smithi038 podman[128164]: 2024-04-26 04:59:35.775606047 +0000 UTC m=+0.837005777 container died 20c115c1cbcc747e812b833795c96df6a5107e381ec08102f6fdf74d52c8d24d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T04:59:36.734 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:36 smithi038 ceph-mon[93172]: pgmap v464: 73 pgs: 3 active+undersized+degraded, 3 peering, 25 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:36.735 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:36 smithi038 ceph-mon[96929]: pgmap v464: 73 pgs: 3 active+undersized+degraded, 3 peering, 25 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:36.735 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:36 smithi038 podman[128301]: 2024-04-26 04:59:36.479589917 +0000 UTC m=+0.697696741 container remove 20c115c1cbcc747e812b833795c96df6a5107e381ec08102f6fdf74d52c8d24d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5) 2024-04-26T04:59:36.735 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:36 smithi038 podman[128365]: 2024-04-26 04:59:36.609927051 +0000 UTC m=+0.019459056 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:36.735 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:36 smithi038 podman[128365]: 2024-04-26 04:59:36.734241227 +0000 UTC m=+0.143773246 container create db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS) 2024-04-26T04:59:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:36 smithi055 ceph-mon[86906]: pgmap v464: 73 pgs: 3 active+undersized+degraded, 3 peering, 25 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:36.987 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:59:36] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.43.0" 2024-04-26T04:59:36.988 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:36 smithi038 podman[128365]: 2024-04-26 04:59:36.983600564 +0000 UTC m=+0.393132552 container init db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T04:59:37.474 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:36 smithi038 podman[128365]: 2024-04-26 04:59:36.987387841 +0000 UTC m=+0.396919823 container start db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS) 2024-04-26T04:59:37.474 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:37 smithi038 ceph-osd[128387]: -- 172.21.15.38:0/3108880651 <== mon.0 v2:172.21.15.38:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563c90083680 con 0x563c9004cc00 2024-04-26T04:59:37.474 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:37 smithi038 bash[128365]: db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e 2024-04-26T04:59:37.474 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:37 smithi038 systemd[1]: Started Ceph osd.3 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T04:59:37.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:37 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T04:59:37.541+0000 7f6d19c557c0 -1 Falling back to public interface 2024-04-26T04:59:38.293 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:38 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:38.293 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:38 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:38.294 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:38 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:38.294 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:38 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3054709617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:38.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3054709617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:38.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:38 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3054709617' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:39.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:39 smithi055 ceph-mon[86906]: pgmap v465: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:39.411 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:39 smithi038 ceph-mon[93172]: pgmap v465: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:39.411 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:39 smithi038 ceph-mon[96929]: pgmap v465: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:40.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:40 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3142124252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:40.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:40 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3142124252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:40.581 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:40 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3142124252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:41.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:41 smithi038 ceph-mon[93172]: pgmap v466: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:41.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:41.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:41 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:41.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:41 smithi038 ceph-mon[96929]: pgmap v466: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:41.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:41.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:41 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:41 smithi055 ceph-mon[86906]: pgmap v466: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:41.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:41.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:41 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3508501283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3508501283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:42 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:42 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3508501283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:42 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:43.724 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T04:59:43.345+0000 7f6d19c557c0 -1 osd.3 127 log_to_monitors true 2024-04-26T04:59:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-mon[93172]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-mon[93172]: from='osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:59:43.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-mon[93172]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:59:43.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-mon[96929]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:43.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-mon[96929]: from='osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:59:43.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-mon[96929]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:59:43.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:43 smithi055 ceph-mon[86906]: pgmap v467: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:43.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:43 smithi055 ceph-mon[86906]: from='osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:59:43.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:43 smithi055 ceph-mon[86906]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-26T04:59:44.164 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 04:59:43 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T04:59:43.886+0000 7f6d070a5700 -1 osd.3 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: osdmap e130: 8 total, 7 up, 8 in 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: from='osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: from='osd.3 ' entity='osd.3' 2024-04-26T04:59:44.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2085537936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: osdmap e130: 8 total, 7 up, 8 in 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: from='osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: from='osd.3 ' entity='osd.3' 2024-04-26T04:59:44.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2085537936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:44.758 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-26T04:59:44.758 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: osdmap e130: 8 total, 7 up, 8 in 2024-04-26T04:59:44.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: from='osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:44.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi038", "root=default"]}]: dispatch 2024-04-26T04:59:44.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-26T04:59:44.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: from='osd.3 ' entity='osd.3' 2024-04-26T04:59:44.759 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2085537936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[93172]: pgmap v469: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[93172]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:59:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[93172]: osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955] boot 2024-04-26T04:59:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[93172]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T04:59:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:45.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[96929]: pgmap v469: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:45.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[96929]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:59:45.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[96929]: osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955] boot 2024-04-26T04:59:45.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[96929]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T04:59:45.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:45 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:45 smithi055 ceph-mon[86906]: pgmap v469: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:45 smithi055 ceph-mon[86906]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-26T04:59:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:45 smithi055 ceph-mon[86906]: osd.3 [v2:172.21.15.38:6826/1402523955,v1:172.21.15.38:6827/1402523955] boot 2024-04-26T04:59:45.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:45 smithi055 ceph-mon[86906]: osdmap e131: 8 total, 8 up, 8 in 2024-04-26T04:59:45.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:45 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-26T04:59:46.473 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T04:59:46.055+0000 7fa1d4887700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:59:46.599 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T04:59:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:59:46] "GET /metrics HTTP/1.1" 200 33730 "" "Prometheus/2.43.0" 2024-04-26T04:59:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: pgmap v471: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T04:59:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: pgmap v473: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: pgmap v474: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2878558380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: pgmap v471: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: pgmap v473: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: pgmap v474: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.226 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:59:47.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:59:47.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:59:47.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2878558380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:47.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.227 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:47.255 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch upgrade pause' 2024-04-26T04:59:47.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: pgmap v471: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: osdmap e132: 8 total, 8 up, 8 in 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: pgmap v473: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: pgmap v474: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-26T04:59:47.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-04-26T04:59:47.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2878558380' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-26T04:59:47.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:47.358 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T04:59:47.537 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:49.073 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:48 smithi038 ceph-mon[93172]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 23 active+undersized, 47 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:49.074 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:48 smithi038 ceph-mon[96929]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 23 active+undersized, 47 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:48 smithi055 ceph-mon[86906]: pgmap v475: 73 pgs: 3 active+undersized+degraded, 23 active+undersized, 47 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-26T04:59:49.557 INFO:teuthology.orchestra.run.smithi038.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:50.337 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:59:50.611 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:50.847 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[93172]: from='client.35103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:50.847 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[93172]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:50.847 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:50.847 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[93172]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:50.848 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[96929]: from='client.35103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:50.848 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[96929]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:50.848 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:50.848 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:50 smithi038 ceph-mon[96929]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:50 smithi055 ceph-mon[86906]: from='client.35103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:50 smithi055 ceph-mon[86906]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T04:59:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:50 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:50 smithi055 ceph-mon[86906]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:51.837 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:51 smithi038 ceph-mon[93172]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-26T04:59:51.837 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:51 smithi038 ceph-mon[96929]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-26T04:59:51.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:51 smithi055 ceph-mon[86906]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-26T04:59:52.629 INFO:teuthology.orchestra.run.smithi038.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:alertmanager.a smithi038 *:9093,9094 running (15m) 11s ago 20m 25.7M - 0.25.0 c8568f914cd2 bd09134fac6b 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:grafana.a smithi055 *:3000 error 11s ago 20m - - 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi038.qsffiu smithi038 running (19m) 11s ago 19m 20.0M - 17.2.7-902-gc61472c8 0de407c1b3af 841ef3f1e866 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi055.cuaegn smithi055 running (19m) 11s ago 19m 15.8M - 17.2.7-902-gc61472c8 0de407c1b3af 030d9ab50e92 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.x smithi055 *:8443,9283,8765 running (14m) 11s ago 24m 455M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f26c41ca1326 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.y smithi038 *:8443,9283,8765 running (14m) 11s ago 26m 512M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4c95c5f5ae02 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mon.a smithi038 running (13m) 11s ago 26m 66.0M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c46affca8e4e 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mon.b smithi055 running (10m) 11s ago 25m 47.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7a992f3a1025 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:mon.c smithi038 running (12m) 11s ago 25m 51.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7e20996682ce 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.a smithi038 *:9100 running (15m) 11s ago 20m 15.0M - 1.5.0 0da6a335fe13 f3d04e42cf8f 2024-04-26T04:59:52.630 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.b smithi055 *:9100 running (15m) 11s ago 20m 15.7M - 1.5.0 0da6a335fe13 9087143d9886 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.0 smithi038 running (8m) 11s ago 24m 780M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 e5183e005cba 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.1 smithi038 running (5m) 11s ago 24m 261M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 71e12d00a8a2 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.2 smithi038 running (52s) 11s ago 23m 41.7M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f3f167abd6fe 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.3 smithi038 running (16s) 11s ago 23m 12.3M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 db6a2a932dd3 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.4 smithi055 running (23m) 11s ago 23m 533M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 40ece9b7ce4f 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.5 smithi055 running (22m) 11s ago 22m 795M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 50fe68fbb905 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.6 smithi055 running (22m) 11s ago 22m 655M 2481M 17.2.7-902-gc61472c8 0de407c1b3af bc76cfe8fbcd 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:osd.7 smithi055 running (21m) 11s ago 21m 588M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 6ca051d18a67 2024-04-26T04:59:52.631 INFO:teuthology.orchestra.run.smithi038.stdout:prometheus.a smithi055 *:9095 running (14m) 11s ago 21m 64.2M - 2.43.0 a07b618ecd1d c40205ac5452 2024-04-26T04:59:52.631 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:52 smithi038 ceph-mon[93172]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:52.632 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:52 smithi038 ceph-mon[96929]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:52.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:52 smithi055 ceph-mon[86906]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:53.423 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph orch ps' 2024-04-26T04:59:53.690 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:53.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:53 smithi038 ceph-mon[93172]: from='client.35109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:53.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:53 smithi038 ceph-mon[96929]: from='client.35109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:53.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:53 smithi055 ceph-mon[86906]: from='client.35109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:54.850 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:54 smithi038 ceph-mon[93172]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:54.850 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:54 smithi038 ceph-mon[96929]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:54.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:54 smithi055 ceph-mon[86906]: pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:alertmanager.a smithi038 *:9093,9094 running (15m) 14s ago 21m 25.7M - 0.25.0 c8568f914cd2 bd09134fac6b 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:grafana.a smithi055 *:3000 error 14s ago 20m - - 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi038.qsffiu smithi038 running (19m) 14s ago 19m 20.0M - 17.2.7-902-gc61472c8 0de407c1b3af 841ef3f1e866 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mds.foo.smithi055.cuaegn smithi055 running (19m) 14s ago 19m 15.8M - 17.2.7-902-gc61472c8 0de407c1b3af 030d9ab50e92 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.x smithi055 *:8443,9283,8765 running (14m) 14s ago 25m 455M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f26c41ca1326 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mgr.y smithi038 *:8443,9283,8765 running (15m) 14s ago 26m 512M - 19.0.0-2447-gb22e2ebd 87e5280d5ae8 4c95c5f5ae02 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mon.a smithi038 running (13m) 14s ago 26m 66.0M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 c46affca8e4e 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mon.b smithi055 running (10m) 14s ago 25m 47.5M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7a992f3a1025 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:mon.c smithi038 running (12m) 14s ago 25m 51.1M 2048M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 7e20996682ce 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.a smithi038 *:9100 running (15m) 14s ago 20m 15.0M - 1.5.0 0da6a335fe13 f3d04e42cf8f 2024-04-26T04:59:55.654 INFO:teuthology.orchestra.run.smithi038.stdout:node-exporter.b smithi055 *:9100 running (15m) 14s ago 20m 15.7M - 1.5.0 0da6a335fe13 9087143d9886 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.0 smithi038 running (8m) 14s ago 24m 780M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 e5183e005cba 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.1 smithi038 running (5m) 14s ago 24m 261M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 71e12d00a8a2 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.2 smithi038 running (55s) 14s ago 23m 41.7M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 f3f167abd6fe 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.3 smithi038 running (19s) 14s ago 23m 12.3M 2481M 19.0.0-2447-gb22e2ebd 87e5280d5ae8 db6a2a932dd3 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.4 smithi055 running (23m) 14s ago 23m 533M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 40ece9b7ce4f 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.5 smithi055 running (22m) 14s ago 22m 795M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 50fe68fbb905 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.6 smithi055 running (22m) 14s ago 22m 655M 2481M 17.2.7-902-gc61472c8 0de407c1b3af bc76cfe8fbcd 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:osd.7 smithi055 running (21m) 14s ago 21m 588M 2481M 17.2.7-902-gc61472c8 0de407c1b3af 6ca051d18a67 2024-04-26T04:59:55.655 INFO:teuthology.orchestra.run.smithi038.stdout:prometheus.a smithi055 *:9095 running (14m) 14s ago 21m 64.2M - 2.43.0 a07b618ecd1d c40205ac5452 2024-04-26T04:59:56.299 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e22c1684-0385-11ef-bc93-c7b262605968 -e sha1=b22e2ebdeb24376882b7bda2a7329c8cccc2276a -- bash -c 'ceph versions' 2024-04-26T04:59:56.573 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 04:59:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:04:59:56] "GET /metrics HTTP/1.1" 200 33730 "" "Prometheus/2.43.0" 2024-04-26T04:59:57.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:56 smithi038 ceph-mon[96929]: from='client.35115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:57.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:56 smithi038 ceph-mon[96929]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:56 smithi038 ceph-mon[93172]: from='client.35115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:57.225 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:56 smithi038 ceph-mon[93172]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:56 smithi055 ceph-mon[86906]: from='client.35115 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-26T04:59:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:56 smithi055 ceph-mon[86906]: pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:58.466 INFO:teuthology.orchestra.run.smithi038.stdout:{ 2024-04-26T04:59:58.466 INFO:teuthology.orchestra.run.smithi038.stdout: "mon": { 2024-04-26T04:59:58.466 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 3 2024-04-26T04:59:58.466 INFO:teuthology.orchestra.run.smithi038.stdout: }, 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "mgr": { 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 2 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: }, 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "osd": { 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 4, 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 4 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: }, 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "mds": { 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 2 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: }, 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "overall": { 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 17.2.7-902-gc61472c8 (c61472c83ce76869b0fe076c19ed2f4bcc3fb4af) quincy (stable)": 6, 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: "ceph version 19.0.0-2447-gb22e2ebd (b22e2ebdeb24376882b7bda2a7329c8cccc2276a) squid (dev)": 9 2024-04-26T04:59:58.467 INFO:teuthology.orchestra.run.smithi038.stdout: } 2024-04-26T04:59:58.468 INFO:teuthology.orchestra.run.smithi038.stdout:} 2024-04-26T04:59:58.479 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:58 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:58.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:58 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:58 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T04:59:59.077 DEBUG:teuthology.parallel:result is None 2024-04-26T04:59:59.077 INFO:teuthology.run_tasks:Running task print... 2024-04-26T04:59:59.086 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-26T04:59:59.086 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-26T04:59:59.217 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-26T04:59:59.217 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-26T04:59:59.218 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-26T04:59:59.489 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T04:59:59.701 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:59 smithi038 ceph-mon[93172]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:59.701 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 04:59:59 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3668000952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:59.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:59 smithi038 ceph-mon[96929]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:59.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 04:59:59 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3668000952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T04:59:59.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:59 smithi055 ceph-mon[86906]: pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T04:59:59.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 04:59:59 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3668000952' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-26T05:00:00.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:00 smithi038 ceph-mon[93172]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:00.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:00 smithi038 ceph-mon[93172]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:00.479 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:00 smithi038 ceph-mon[93172]: daemon grafana.a on smithi055 is in error state 2024-04-26T05:00:00.480 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:00 smithi038 ceph-mon[96929]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:00.480 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:00 smithi038 ceph-mon[96929]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:00.480 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:00 smithi038 ceph-mon[96929]: daemon grafana.a on smithi055 is in error state 2024-04-26T05:00:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:00 smithi055 ceph-mon[86906]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-26T05:00:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:00 smithi055 ceph-mon[86906]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-26T05:00:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:00 smithi055 ceph-mon[86906]: daemon grafana.a on smithi055 is in error state 2024-04-26T05:00:01.452 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:01.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:00:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:02.093 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":39,"flags":0,"active_gid":25231,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6800","nonce":2674007905},{"type":"v1","addr":"172.21.15.38:6801","nonce":2674007905}]},"active_addr":"172.21.15.38:6801/2674007905","active_change":"2024-04-26T04:46:16.590369+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25228,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.38:8443/","prometheus":"http://172.21.15.38:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":82,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":1428450442}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":2658251416}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":3660766384}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.38:0","nonce":2240183553}]}]} 2024-04-26T05:00:02.097 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-26T05:00:02.097 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-26T05:00:02.097 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:00:02.366 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/927155713' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/927155713' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:02.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:02.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: pgmap v484: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: pgmap v485: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/927155713' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-26T05:00:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:00:02.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:04 smithi038 ceph-mon[93172]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:04.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:04 smithi038 ceph-mon[96929]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:04.509 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:04.509 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":132,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:59:45.880694+0000","last_up_change":"2024-04-26T04:59:44.878459+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:38:52.695557+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:39:55.221857+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:39:55.718635+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":131,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6803","nonce":3874109320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6805","nonce":3874109320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6809","nonce":3874109320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6807","nonce":3874109320}]},"public_addr":"172.21.15.38:6803/3874109320","cluster_addr":"172.21.15.38:6805/3874109320","heartbeat_back_addr":"172.21.15.38:6809/3874109320","heartbeat_front_addr":"172.21.15.38:6807/3874109320","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":109,"up_from":113,"up_thru":131,"down_at":110,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6811","nonce":2413675931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6813","nonce":2413675931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6817","nonce":2413675931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6815","nonce":2413675931}]},"public_addr":"172.21.15.38:6811/2413675931","cluster_addr":"172.21.15.38:6813/2413675931","heartbeat_back_addr":"172.21.15.38:6817/2413675931","heartbeat_front_addr":"172.21.15.38:6815/2413675931","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":126,"up_thru":131,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6819","nonce":3191109599}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6821","nonce":3191109599}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6825","nonce":3191109599}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6823","nonce":3191109599}]},"public_addr":"172.21.15.38:6819/3191109599","cluster_addr":"172.21.15.38:6821/3191109599","heartbeat_back_addr":"172.21.15.38:6825/3191109599","heartbeat_front_addr":"172.21.15.38:6823/3191109599","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":127,"up_from":131,"up_thru":131,"down_at":128,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6827","nonce":1402523955}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6829","nonce":1402523955}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6833","nonce":1402523955}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6831","nonce":1402523955}]},"public_addr":"172.21.15.38:6827/1402523955","cluster_addr":"172.21.15.38:6829/1402523955","heartbeat_back_addr":"172.21.15.38:6833/1402523955","heartbeat_front_addr":"172.21.15.38:6831/1402523955","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":128,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":109},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":127},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.38:0/3579807211":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/3928681912":"2024-04-27T04:46:16.590082+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/3488049706":"2024-04-27T04:43:39.757061+0000","172.21.15.55:0/2237373725":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.55:0/1655304218":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/1564621640":"2024-04-27T04:43:39.757061+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.55:6834/1712502054":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/4062968598":"2024-04-27T04:43:39.757061+0000","172.21.15.55:0/95854323":"2024-04-27T04:45:08.526108+0000","172.21.15.55:6835/1712502054":"2024-04-27T04:45:08.526108+0000","172.21.15.55:0/3117976104":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/3149555040":"2024-04-27T04:43:39.757061+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3362510910":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/1699336096":"2024-04-27T04:43:39.757061+0000","172.21.15.38:6800/1054120581":"2024-04-27T04:46:16.590082+0000","172.21.15.55:0/893057966":"2024-04-27T04:45:08.526108+0000","172.21.15.38:6800/3488049706":"2024-04-27T04:43:39.757061+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/3761520179":"2024-04-27T04:46:16.590082+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.55:0/1688421475":"2024-04-27T04:45:08.526108+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2047784554":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2814321010":"2024-04-27T04:46:16.590082+0000","172.21.15.38:6801/1054120581":"2024-04-27T04:46:16.590082+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:00:04.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:04 smithi055 ceph-mon[86906]: pgmap v486: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:05.150 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-26T05:00:05.150 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-26T05:00:05.418 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:05.662 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:05 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3425492850' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:00:05.663 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:05 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3425492850' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:00:05.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:05 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3425492850' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:00:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:06 smithi038 ceph-mon[93172]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:06 smithi038 ceph-mon[96929]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:06.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:06 smithi055 ceph-mon[86906]: pgmap v487: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:07.051 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:00:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:00:06] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:00:07.271 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:07.271 INFO:teuthology.orchestra.run.smithi038.stdout:{"epoch":132,"fsid":"e22c1684-0385-11ef-bc93-c7b262605968","created":"2024-04-26T04:32:59.901779+0000","modified":"2024-04-26T04:59:45.880694+0000","last_up_change":"2024-04-26T04:59:44.878459+0000","last_in_change":"2024-04-26T04:37:50.753043+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-26T04:36:10.250717+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-26T04:38:52.695557+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-26T04:39:55.221857+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-26T04:39:55.718635+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"01bfbe1d-b610-4e51-bebd-99416d020dab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":87,"up_from":91,"up_thru":131,"down_at":88,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6802","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6803","nonce":3874109320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6804","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6805","nonce":3874109320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6808","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6809","nonce":3874109320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6806","nonce":3874109320},{"type":"v1","addr":"172.21.15.38:6807","nonce":3874109320}]},"public_addr":"172.21.15.38:6803/3874109320","cluster_addr":"172.21.15.38:6805/3874109320","heartbeat_back_addr":"172.21.15.38:6809/3874109320","heartbeat_front_addr":"172.21.15.38:6807/3874109320","state":["exists","up"]},{"osd":1,"uuid":"7f8dd4ef-f2b7-41af-8cbd-26faa274219a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":109,"up_from":113,"up_thru":131,"down_at":110,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6810","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6811","nonce":2413675931}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6812","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6813","nonce":2413675931}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6816","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6817","nonce":2413675931}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6814","nonce":2413675931},{"type":"v1","addr":"172.21.15.38:6815","nonce":2413675931}]},"public_addr":"172.21.15.38:6811/2413675931","cluster_addr":"172.21.15.38:6813/2413675931","heartbeat_back_addr":"172.21.15.38:6817/2413675931","heartbeat_front_addr":"172.21.15.38:6815/2413675931","state":["exists","up"]},{"osd":2,"uuid":"ed88ae78-f042-46aa-af4a-8e62ed88e391","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":126,"up_thru":131,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6818","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6819","nonce":3191109599}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6820","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6821","nonce":3191109599}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6824","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6825","nonce":3191109599}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6822","nonce":3191109599},{"type":"v1","addr":"172.21.15.38:6823","nonce":3191109599}]},"public_addr":"172.21.15.38:6819/3191109599","cluster_addr":"172.21.15.38:6821/3191109599","heartbeat_back_addr":"172.21.15.38:6825/3191109599","heartbeat_front_addr":"172.21.15.38:6823/3191109599","state":["exists","up"]},{"osd":3,"uuid":"91b59c6c-a9ca-469d-b8a5-e966b283ef3b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":127,"up_from":131,"up_thru":131,"down_at":128,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6826","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6827","nonce":1402523955}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6828","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6829","nonce":1402523955}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6832","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6833","nonce":1402523955}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.38:6830","nonce":1402523955},{"type":"v1","addr":"172.21.15.38:6831","nonce":1402523955}]},"public_addr":"172.21.15.38:6827/1402523955","cluster_addr":"172.21.15.38:6829/1402523955","heartbeat_back_addr":"172.21.15.38:6833/1402523955","heartbeat_front_addr":"172.21.15.38:6831/1402523955","state":["exists","up"]},{"osd":4,"uuid":"d2c7373f-2b75-4f2e-b028-d5c1dabcee4a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6801","nonce":3196578414}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6803","nonce":3196578414}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6807","nonce":3196578414}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3196578414},{"type":"v1","addr":"172.21.15.55:6805","nonce":3196578414}]},"public_addr":"172.21.15.55:6801/3196578414","cluster_addr":"172.21.15.55:6803/3196578414","heartbeat_back_addr":"172.21.15.55:6807/3196578414","heartbeat_front_addr":"172.21.15.55:6805/3196578414","state":["exists","up"]},{"osd":5,"uuid":"fddae5a2-7bb9-417d-a4a0-f8bd5aef226a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6809","nonce":801519512}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6811","nonce":801519512}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6815","nonce":801519512}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":801519512},{"type":"v1","addr":"172.21.15.55:6813","nonce":801519512}]},"public_addr":"172.21.15.55:6809/801519512","cluster_addr":"172.21.15.55:6811/801519512","heartbeat_back_addr":"172.21.15.55:6815/801519512","heartbeat_front_addr":"172.21.15.55:6813/801519512","state":["exists","up"]},{"osd":6,"uuid":"7d114339-5312-4b95-bb3e-b3a68a38d683","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":128,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6817","nonce":1074490219}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6819","nonce":1074490219}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6823","nonce":1074490219}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1074490219},{"type":"v1","addr":"172.21.15.55:6821","nonce":1074490219}]},"public_addr":"172.21.15.55:6817/1074490219","cluster_addr":"172.21.15.55:6819/1074490219","heartbeat_back_addr":"172.21.15.55:6823/1074490219","heartbeat_front_addr":"172.21.15.55:6821/1074490219","state":["exists","up"]},{"osd":7,"uuid":"55f50e10-51c3-4688-bd17-c0abdb973358","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":131,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6825","nonce":1950122653}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6827","nonce":1950122653}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6831","nonce":1950122653}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":1950122653},{"type":"v1","addr":"172.21.15.55:6829","nonce":1950122653}]},"public_addr":"172.21.15.55:6825/1950122653","cluster_addr":"172.21.15.55:6827/1950122653","heartbeat_back_addr":"172.21.15.55:6831/1950122653","heartbeat_front_addr":"172.21.15.55:6829/1950122653","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:16.872079+0000","dead_epoch":87},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:35:41.726706+0000","dead_epoch":109},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:05.957577+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:32.294100+0000","dead_epoch":127},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:36:54.286135+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:18.143574+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:37:42.012012+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-26T04:38:05.916695+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/95854323":"2024-04-27T04:45:08.526108+0000","172.21.15.38:6801/3488049706":"2024-04-27T04:43:39.757061+0000","172.21.15.55:0/2237373725":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/3362510910":"2024-04-27T04:46:16.590082+0000","172.21.15.38:6800/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/3092409138":"2024-04-27T04:33:31.882245+0000","172.21.15.55:0/1655304218":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/3279184424":"2024-04-27T04:38:38.949605+0000","172.21.15.55:6834/1712502054":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/4062968598":"2024-04-27T04:43:39.757061+0000","172.21.15.55:6835/1712502054":"2024-04-27T04:45:08.526108+0000","172.21.15.55:0/3117976104":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/3149555040":"2024-04-27T04:43:39.757061+0000","172.21.15.38:6800/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/1511225059":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1876651557":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3761520179":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/2419990728":"2024-04-27T04:33:31.882245+0000","172.21.15.38:6801/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/1564621640":"2024-04-27T04:43:39.757061+0000","172.21.15.38:0/21921625":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2047784554":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/935836958":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/2814321010":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/2062166906":"2024-04-27T04:38:38.949605+0000","172.21.15.38:0/2957744378":"2024-04-27T04:33:55.739521+0000","172.21.15.38:6800/170598977":"2024-04-27T04:33:31.882245+0000","172.21.15.55:0/1688421475":"2024-04-27T04:45:08.526108+0000","172.21.15.38:6801/1160588417":"2024-04-27T04:38:38.949605+0000","172.21.15.38:6801/1054120581":"2024-04-27T04:46:16.590082+0000","172.21.15.38:0/3017786259":"2024-04-27T04:33:31.882245+0000","172.21.15.38:0/1699336096":"2024-04-27T04:43:39.757061+0000","172.21.15.38:6800/1054120581":"2024-04-27T04:46:16.590082+0000","172.21.15.55:0/893057966":"2024-04-27T04:45:08.526108+0000","172.21.15.38:0/3928681912":"2024-04-27T04:46:16.590082+0000","172.21.15.38:6800/3488049706":"2024-04-27T04:43:39.757061+0000","172.21.15.38:0/3585961394":"2024-04-27T04:33:55.739521+0000","172.21.15.38:0/3579807211":"2024-04-27T04:46:16.590082+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-26T05:00:07.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/460480306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:00:07.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/460480306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:00:07.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/460480306' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-26T05:00:07.891 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-26T05:00:07.891 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-26T05:00:07.892 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-26T05:00:07.892 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-26T05:00:07.893 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-26T05:00:07.893 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-26T05:00:07.893 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-26T05:00:07.894 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-26T05:00:08.490 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.492 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.492 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.496 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.496 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.500 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.500 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.501 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:08.638 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:08 smithi038 ceph-mon[93172]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:08.638 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:08 smithi038 ceph-mon[96929]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:08.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:08 smithi055 ceph-mon[86906]: pgmap v488: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.715 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:10 smithi038 ceph-mon[93172]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.715 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:10 smithi038 ceph-mon[96929]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:10.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:10 smithi055 ceph-mon[86906]: pgmap v489: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:12 smithi038 ceph-mon[93172]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:12 smithi038 ceph-mon[96929]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:12.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:12 smithi055 ceph-mon[86906]: pgmap v490: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.603 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:14 smithi038 ceph-mon[93172]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.603 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:14 smithi038 ceph-mon[96929]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:14.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:14 smithi055 ceph-mon[86906]: pgmap v491: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:16 smithi038 ceph-mon[96929]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:16 smithi038 ceph-mon[93172]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:16.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:16 smithi055 ceph-mon[86906]: pgmap v492: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:00:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:00:16] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:00:17.686 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.687 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:17.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:18.654 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:18 smithi038 ceph-mon[93172]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.655 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:18 smithi038 ceph-mon[96929]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:18 smithi055 ceph-mon[86906]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:19.401 INFO:teuthology.orchestra.run.smithi038.stdout:541165879311 2024-04-26T05:00:19.401 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-26T05:00:19.670 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:19 smithi038 ceph-mon[93172]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:19.670 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:19 smithi038 ceph-mon[96929]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:19 smithi055 ceph-mon[86906]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:21.305 INFO:teuthology.orchestra.run.smithi038.stdout:485331304515 2024-04-26T05:00:21.306 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-26T05:00:21.964 INFO:teuthology.orchestra.run.smithi038.stdout:562640715784 2024-04-26T05:00:21.965 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-26T05:00:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:22 smithi055 ceph-mon[86906]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:22.623 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:22 smithi038 ceph-mon[96929]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:22.623 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:22 smithi038 ceph-mon[93172]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:22.977 INFO:teuthology.orchestra.run.smithi038.stdout:390842024042 2024-04-26T05:00:22.977 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-26T05:00:23.714 INFO:teuthology.orchestra.run.smithi038.stdout:167503724819 2024-04-26T05:00:23.714 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-26T05:00:24.207 INFO:teuthology.orchestra.run.smithi038.stdout:133143986457 2024-04-26T05:00:24.208 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-26T05:00:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:24 smithi038 ceph-mon[93172]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:24.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:24 smithi038 ceph-mon[96929]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:24 smithi055 ceph-mon[86906]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:24.829 INFO:teuthology.orchestra.run.smithi038.stdout:236223201548 2024-04-26T05:00:24.829 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-26T05:00:25.003 INFO:teuthology.orchestra.run.smithi038.stdout:201863463184 2024-04-26T05:00:25.003 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-26T05:00:25.049 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:25.050 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:25.386 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:25.387 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:25.387 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:25.388 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:25.815 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:26.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:26 smithi038 ceph-mon[93172]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:26.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:26 smithi038 ceph-mon[96929]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:26.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:26 smithi055 ceph-mon[86906]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:00:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:00:26] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:00:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:28 smithi055 ceph-mon[86906]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:28.613 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:28 smithi038 ceph-mon[93172]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:28.613 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:28 smithi038 ceph-mon[96929]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:30.573 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:30 smithi038 ceph-mon[96929]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:30.573 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:30 smithi038 ceph-mon[93172]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:30.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:30 smithi055 ceph-mon[86906]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:30.911 INFO:teuthology.orchestra.run.smithi038.stdout:562640715787 2024-04-26T05:00:31.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2220763198' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:00:31.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2220763198' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:00:31.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2220763198' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-26T05:00:31.813 INFO:tasks.cephadm.ceph_manager.ceph:need seq 562640715784 got 562640715787 for osd.3 2024-04-26T05:00:31.814 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:32.226 INFO:teuthology.orchestra.run.smithi038.stdout:167503724822 2024-04-26T05:00:32.560 INFO:teuthology.orchestra.run.smithi038.stdout:390842024044 2024-04-26T05:00:32.561 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:32 smithi038 ceph-mon[93172]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:32.561 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:32.561 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:32 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1565026351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:00:32.561 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:32 smithi038 ceph-mon[96929]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:32.561 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:32.561 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:32 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1565026351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:00:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:32 smithi055 ceph-mon[86906]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:32 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1565026351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-26T05:00:33.183 INFO:tasks.cephadm.ceph_manager.ceph:need seq 390842024042 got 390842024044 for osd.0 2024-04-26T05:00:33.183 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:33.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:33 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1102566877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:00:33.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:33 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1102566877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:00:33.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:33 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1102566877' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-26T05:00:33.626 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724819 got 167503724822 for osd.5 2024-04-26T05:00:33.627 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:34.235 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:34.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:34 smithi038 ceph-mon[93172]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:34 smithi038 ceph-mon[96929]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:34 smithi055 ceph-mon[86906]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:35.725 INFO:teuthology.orchestra.run.smithi038.stdout:485331304518 2024-04-26T05:00:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:36 smithi038 ceph-mon[93172]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:36.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/754531972' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:00:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:36 smithi038 ceph-mon[96929]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:36.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/754531972' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:00:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:36 smithi055 ceph-mon[86906]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/754531972' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-26T05:00:36.728 INFO:tasks.cephadm.ceph_manager.ceph:need seq 485331304515 got 485331304518 for osd.1 2024-04-26T05:00:36.729 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:36.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:00:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:00:36] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:00:37.580 INFO:teuthology.orchestra.run.smithi038.stdout:133143986461 2024-04-26T05:00:38.637 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:38 smithi038 ceph-mon[96929]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.637 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:38 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1472249121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:00:38.637 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:38 smithi038 ceph-mon[93172]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.638 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:38 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1472249121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:00:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:38 smithi055 ceph-mon[86906]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:38 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1472249121' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-26T05:00:39.839 INFO:teuthology.orchestra.run.smithi038.stdout:201863463188 2024-04-26T05:00:40.015 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986457 got 133143986461 for osd.4 2024-04-26T05:00:40.015 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:40.619 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:40 smithi038 ceph-mon[93172]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:40.619 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:40 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/589083739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:00:40.619 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:40 smithi038 ceph-mon[96929]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:40.620 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:40 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/589083739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:00:40.650 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463184 got 201863463188 for osd.6 2024-04-26T05:00:40.650 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:40 smithi055 ceph-mon[86906]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:40 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/589083739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-26T05:00:41.322 INFO:teuthology.orchestra.run.smithi038.stdout:541165879316 2024-04-26T05:00:41.514 INFO:teuthology.orchestra.run.smithi038.stdout:236223201551 2024-04-26T05:00:41.561 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:41 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3393836694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:00:41.562 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:41 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3393836694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:00:41.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:41 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3393836694' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-26T05:00:42.120 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201548 got 236223201551 for osd.7 2024-04-26T05:00:42.120 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:42.633 INFO:tasks.cephadm.ceph_manager.ceph:need seq 541165879311 got 541165879316 for osd.2 2024-04-26T05:00:42.633 DEBUG:teuthology.parallel:result is None 2024-04-26T05:00:42.633 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-26T05:00:42.634 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:00:42.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:42 smithi038 ceph-mon[93172]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:42.658 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:42 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2105246327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:00:42.658 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:42 smithi038 ceph-mon[96929]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:42.658 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:42 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2105246327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:00:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:42 smithi055 ceph-mon[86906]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:42 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2105246327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-26T05:00:42.902 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:44.679 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:44.680 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:44 smithi038 ceph-mon[96929]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:44.680 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:44 smithi038 ceph-mon[93172]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:44.683 INFO:teuthology.orchestra.run.smithi038.stderr:dumped all 2024-04-26T05:00:44.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:44 smithi055 ceph-mon[86906]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:45.447 INFO:teuthology.orchestra.run.smithi038.stdout:{"pg_ready":true,"pg_map":{"version":506,"stamp":"2024-04-26T05:00:43.233675+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2708112,"kb_used_data":13908,"kb_used_omap":29,"kb_used_meta":2694050,"kb_avail":747253104,"statfs":{"total":767960285184,"available":765187178496,"internally_reserved":0,"allocated":14241792,"data_stored":8463758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30104,"internal_metadata":2758707816},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003012"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":110,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.897873+0000","last_change":"2024-04-26T04:59:09.197430+0000","last_active":"2024-04-26T04:59:44.897873+0000","last_peered":"2024-04-26T04:59:44.897873+0000","last_clean":"2024-04-26T04:59:44.897873+0000","last_became_active":"2024-04-26T04:59:09.197082+0000","last_became_peered":"2024-04-26T04:59:09.197082+0000","last_unstale":"2024-04-26T04:59:44.897873+0000","last_undegraded":"2024-04-26T04:59:44.897873+0000","last_fullsized":"2024-04-26T04:59:44.897873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:02:36.345011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903180+0000","last_change":"2024-04-26T04:59:45.903180+0000","last_active":"2024-04-26T04:59:45.903180+0000","last_peered":"2024-04-26T04:59:45.903180+0000","last_clean":"2024-04-26T04:59:45.903180+0000","last_became_active":"2024-04-26T04:59:45.902801+0000","last_became_peered":"2024-04-26T04:59:45.902801+0000","last_unstale":"2024-04-26T04:59:45.903180+0000","last_undegraded":"2024-04-26T04:59:45.903180+0000","last_fullsized":"2024-04-26T04:59:45.903180+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:36.071038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896797+0000","last_change":"2024-04-26T04:59:45.896797+0000","last_active":"2024-04-26T04:59:45.896797+0000","last_peered":"2024-04-26T04:59:45.896797+0000","last_clean":"2024-04-26T04:59:45.896797+0000","last_became_active":"2024-04-26T04:59:45.896456+0000","last_became_peered":"2024-04-26T04:59:45.896456+0000","last_unstale":"2024-04-26T04:59:45.896797+0000","last_undegraded":"2024-04-26T04:59:45.896797+0000","last_fullsized":"2024-04-26T04:59:45.896797+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:23:19.527315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905441+0000","last_change":"2024-04-26T04:59:45.905441+0000","last_active":"2024-04-26T04:59:45.905441+0000","last_peered":"2024-04-26T04:59:45.905441+0000","last_clean":"2024-04-26T04:59:45.905441+0000","last_became_active":"2024-04-26T04:59:45.905077+0000","last_became_peered":"2024-04-26T04:59:45.905077+0000","last_unstale":"2024-04-26T04:59:45.905441+0000","last_undegraded":"2024-04-26T04:59:45.905441+0000","last_fullsized":"2024-04-26T04:59:45.905441+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:01:03.311399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897272+0000","last_change":"2024-04-26T04:59:45.897272+0000","last_active":"2024-04-26T04:59:45.897272+0000","last_peered":"2024-04-26T04:59:45.897272+0000","last_clean":"2024-04-26T04:59:45.897272+0000","last_became_active":"2024-04-26T04:59:45.896838+0000","last_became_peered":"2024-04-26T04:59:45.896838+0000","last_unstale":"2024-04-26T04:59:45.897272+0000","last_undegraded":"2024-04-26T04:59:45.897272+0000","last_fullsized":"2024-04-26T04:59:45.897272+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:00:06.719287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895326+0000","last_change":"2024-04-26T04:59:09.198697+0000","last_active":"2024-04-26T04:59:44.895326+0000","last_peered":"2024-04-26T04:59:44.895326+0000","last_clean":"2024-04-26T04:59:44.895326+0000","last_became_active":"2024-04-26T04:59:09.198364+0000","last_became_peered":"2024-04-26T04:59:09.198364+0000","last_unstale":"2024-04-26T04:59:44.895326+0000","last_undegraded":"2024-04-26T04:59:44.895326+0000","last_fullsized":"2024-04-26T04:59:44.895326+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:52.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906493+0000","last_change":"2024-04-26T04:59:45.906493+0000","last_active":"2024-04-26T04:59:45.906493+0000","last_peered":"2024-04-26T04:59:45.906493+0000","last_clean":"2024-04-26T04:59:45.906493+0000","last_became_active":"2024-04-26T04:59:45.906198+0000","last_became_peered":"2024-04-26T04:59:45.906198+0000","last_unstale":"2024-04-26T04:59:45.906493+0000","last_undegraded":"2024-04-26T04:59:45.906493+0000","last_fullsized":"2024-04-26T04:59:45.906493+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:12:50.480125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.898046+0000","last_change":"2024-04-26T04:59:45.898046+0000","last_active":"2024-04-26T04:59:45.898046+0000","last_peered":"2024-04-26T04:59:45.898046+0000","last_clean":"2024-04-26T04:59:45.898046+0000","last_became_active":"2024-04-26T04:59:45.897507+0000","last_became_peered":"2024-04-26T04:59:45.897507+0000","last_unstale":"2024-04-26T04:59:45.898046+0000","last_undegraded":"2024-04-26T04:59:45.898046+0000","last_fullsized":"2024-04-26T04:59:45.898046+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:45:23.980125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889543+0000","last_change":"2024-04-26T04:54:53.653738+0000","last_active":"2024-04-26T04:59:44.889543+0000","last_peered":"2024-04-26T04:59:44.889543+0000","last_clean":"2024-04-26T04:59:44.889543+0000","last_became_active":"2024-04-26T04:54:53.653382+0000","last_became_peered":"2024-04-26T04:54:53.653382+0000","last_unstale":"2024-04-26T04:59:44.889543+0000","last_undegraded":"2024-04-26T04:59:44.889543+0000","last_fullsized":"2024-04-26T04:59:44.889543+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:32:37.322919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903161+0000","last_change":"2024-04-26T04:59:45.903161+0000","last_active":"2024-04-26T04:59:45.903161+0000","last_peered":"2024-04-26T04:59:45.903161+0000","last_clean":"2024-04-26T04:59:45.903161+0000","last_became_active":"2024-04-26T04:59:45.902756+0000","last_became_peered":"2024-04-26T04:59:45.902756+0000","last_unstale":"2024-04-26T04:59:45.903161+0000","last_undegraded":"2024-04-26T04:59:45.903161+0000","last_fullsized":"2024-04-26T04:59:45.903161+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:24:02.116202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":95,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905512+0000","last_change":"2024-04-26T04:59:45.905512+0000","last_active":"2024-04-26T04:59:45.905512+0000","last_peered":"2024-04-26T04:59:45.905512+0000","last_clean":"2024-04-26T04:59:45.905512+0000","last_became_active":"2024-04-26T04:59:45.905117+0000","last_became_peered":"2024-04-26T04:59:45.905117+0000","last_unstale":"2024-04-26T04:59:45.905512+0000","last_undegraded":"2024-04-26T04:59:45.905512+0000","last_fullsized":"2024-04-26T04:59:45.905512+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:38:53.980299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887699+0000","last_change":"2024-04-26T04:59:09.194955+0000","last_active":"2024-04-26T04:59:45.887699+0000","last_peered":"2024-04-26T04:59:45.887699+0000","last_clean":"2024-04-26T04:59:45.887699+0000","last_became_active":"2024-04-26T04:59:09.194612+0000","last_became_peered":"2024-04-26T04:59:09.194612+0000","last_unstale":"2024-04-26T04:59:45.887699+0000","last_undegraded":"2024-04-26T04:59:45.887699+0000","last_fullsized":"2024-04-26T04:59:45.887699+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:57:07.292445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":97,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895513+0000","last_change":"2024-04-26T04:39:56.300576+0000","last_active":"2024-04-26T04:59:44.895513+0000","last_peered":"2024-04-26T04:59:44.895513+0000","last_clean":"2024-04-26T04:59:44.895513+0000","last_became_active":"2024-04-26T04:39:56.299260+0000","last_became_peered":"2024-04-26T04:39:56.299260+0000","last_unstale":"2024-04-26T04:59:44.895513+0000","last_undegraded":"2024-04-26T04:59:44.895513+0000","last_fullsized":"2024-04-26T04:59:44.895513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:26.586898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.898743+0000","last_change":"2024-04-26T04:54:53.663678+0000","last_active":"2024-04-26T04:59:44.898743+0000","last_peered":"2024-04-26T04:59:44.898743+0000","last_clean":"2024-04-26T04:59:44.898743+0000","last_became_active":"2024-04-26T04:54:53.662856+0000","last_became_peered":"2024-04-26T04:54:53.662856+0000","last_unstale":"2024-04-26T04:59:44.898743+0000","last_undegraded":"2024-04-26T04:59:44.898743+0000","last_fullsized":"2024-04-26T04:59:44.898743+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:50:16.153714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.892961+0000","last_change":"2024-04-26T04:59:09.201573+0000","last_active":"2024-04-26T04:59:44.892961+0000","last_peered":"2024-04-26T04:59:44.892961+0000","last_clean":"2024-04-26T04:59:44.892961+0000","last_became_active":"2024-04-26T04:59:09.201166+0000","last_became_peered":"2024-04-26T04:59:09.201166+0000","last_unstale":"2024-04-26T04:59:44.892961+0000","last_undegraded":"2024-04-26T04:59:44.892961+0000","last_fullsized":"2024-04-26T04:59:44.892961+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:39:45.735710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903984+0000","last_change":"2024-04-26T04:59:45.903984+0000","last_active":"2024-04-26T04:59:45.903984+0000","last_peered":"2024-04-26T04:59:45.903984+0000","last_clean":"2024-04-26T04:59:45.903984+0000","last_became_active":"2024-04-26T04:59:45.902811+0000","last_became_peered":"2024-04-26T04:59:45.902811+0000","last_unstale":"2024-04-26T04:59:45.903984+0000","last_undegraded":"2024-04-26T04:59:45.903984+0000","last_fullsized":"2024-04-26T04:59:45.903984+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:57:45.118873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905936+0000","last_change":"2024-04-26T04:59:45.905936+0000","last_active":"2024-04-26T04:59:45.905936+0000","last_peered":"2024-04-26T04:59:45.905936+0000","last_clean":"2024-04-26T04:59:45.905936+0000","last_became_active":"2024-04-26T04:59:45.905272+0000","last_became_peered":"2024-04-26T04:59:45.905272+0000","last_unstale":"2024-04-26T04:59:45.905936+0000","last_undegraded":"2024-04-26T04:59:45.905936+0000","last_fullsized":"2024-04-26T04:59:45.905936+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:39:27.886854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.898668+0000","last_change":"2024-04-26T04:59:45.898668+0000","last_active":"2024-04-26T04:59:45.898668+0000","last_peered":"2024-04-26T04:59:45.898668+0000","last_clean":"2024-04-26T04:59:45.898668+0000","last_became_active":"2024-04-26T04:59:45.898338+0000","last_became_peered":"2024-04-26T04:59:45.898338+0000","last_unstale":"2024-04-26T04:59:45.898668+0000","last_undegraded":"2024-04-26T04:59:45.898668+0000","last_fullsized":"2024-04-26T04:59:45.898668+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:39.484941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":75,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891626+0000","last_change":"2024-04-26T04:39:56.302393+0000","last_active":"2024-04-26T04:59:44.891626+0000","last_peered":"2024-04-26T04:59:44.891626+0000","last_clean":"2024-04-26T04:59:44.891626+0000","last_became_active":"2024-04-26T04:39:56.302116+0000","last_became_peered":"2024-04-26T04:39:56.302116+0000","last_unstale":"2024-04-26T04:59:44.891626+0000","last_undegraded":"2024-04-26T04:59:44.891626+0000","last_fullsized":"2024-04-26T04:59:44.891626+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:28:13.475580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905982+0000","last_change":"2024-04-26T04:59:45.905982+0000","last_active":"2024-04-26T04:59:45.905982+0000","last_peered":"2024-04-26T04:59:45.905982+0000","last_clean":"2024-04-26T04:59:45.905982+0000","last_became_active":"2024-04-26T04:59:45.905444+0000","last_became_peered":"2024-04-26T04:59:45.905444+0000","last_unstale":"2024-04-26T04:59:45.905982+0000","last_undegraded":"2024-04-26T04:59:45.905982+0000","last_fullsized":"2024-04-26T04:59:45.905982+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:27:03.195942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":100,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895717+0000","last_change":"2024-04-26T04:51:40.691367+0000","last_active":"2024-04-26T04:59:44.895717+0000","last_peered":"2024-04-26T04:59:44.895717+0000","last_clean":"2024-04-26T04:59:44.895717+0000","last_became_active":"2024-04-26T04:51:40.690968+0000","last_became_peered":"2024-04-26T04:51:40.690968+0000","last_unstale":"2024-04-26T04:59:44.895717+0000","last_undegraded":"2024-04-26T04:59:44.895717+0000","last_fullsized":"2024-04-26T04:59:44.895717+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:51:00.577555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905593+0000","last_change":"2024-04-26T04:59:45.905593+0000","last_active":"2024-04-26T04:59:45.905593+0000","last_peered":"2024-04-26T04:59:45.905593+0000","last_clean":"2024-04-26T04:59:45.905593+0000","last_became_active":"2024-04-26T04:59:45.904348+0000","last_became_peered":"2024-04-26T04:59:45.904348+0000","last_unstale":"2024-04-26T04:59:45.905593+0000","last_undegraded":"2024-04-26T04:59:45.905593+0000","last_fullsized":"2024-04-26T04:59:45.905593+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:00:33.450842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":93,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.893228+0000","last_change":"2024-04-26T04:51:40.688923+0000","last_active":"2024-04-26T04:59:44.893228+0000","last_peered":"2024-04-26T04:59:44.893228+0000","last_clean":"2024-04-26T04:59:44.893228+0000","last_became_active":"2024-04-26T04:51:40.688072+0000","last_became_peered":"2024-04-26T04:51:40.688072+0000","last_unstale":"2024-04-26T04:59:44.893228+0000","last_undegraded":"2024-04-26T04:59:44.893228+0000","last_fullsized":"2024-04-26T04:59:44.893228+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:35:37.600512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889272+0000","last_change":"2024-04-26T04:39:56.925632+0000","last_active":"2024-04-26T04:59:44.889272+0000","last_peered":"2024-04-26T04:59:44.889272+0000","last_clean":"2024-04-26T04:59:44.889272+0000","last_became_active":"2024-04-26T04:39:56.925327+0000","last_became_peered":"2024-04-26T04:39:56.925327+0000","last_unstale":"2024-04-26T04:59:44.889272+0000","last_undegraded":"2024-04-26T04:59:44.889272+0000","last_fullsized":"2024-04-26T04:59:44.889272+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:29:59.594965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.893141+0000","last_change":"2024-04-26T04:54:53.654360+0000","last_active":"2024-04-26T04:59:44.893141+0000","last_peered":"2024-04-26T04:59:44.893141+0000","last_clean":"2024-04-26T04:59:44.893141+0000","last_became_active":"2024-04-26T04:54:53.653748+0000","last_became_peered":"2024-04-26T04:54:53.653748+0000","last_unstale":"2024-04-26T04:59:44.893141+0000","last_undegraded":"2024-04-26T04:59:44.893141+0000","last_fullsized":"2024-04-26T04:59:44.893141+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:32:03.616453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900765+0000","last_change":"2024-04-26T04:59:45.900765+0000","last_active":"2024-04-26T04:59:45.900765+0000","last_peered":"2024-04-26T04:59:45.900765+0000","last_clean":"2024-04-26T04:59:45.900765+0000","last_became_active":"2024-04-26T04:59:45.900356+0000","last_became_peered":"2024-04-26T04:59:45.900356+0000","last_unstale":"2024-04-26T04:59:45.900765+0000","last_undegraded":"2024-04-26T04:59:45.900765+0000","last_fullsized":"2024-04-26T04:59:45.900765+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:35:37.891548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":76,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891292+0000","last_change":"2024-04-26T04:39:56.300360+0000","last_active":"2024-04-26T04:59:44.891292+0000","last_peered":"2024-04-26T04:59:44.891292+0000","last_clean":"2024-04-26T04:59:44.891292+0000","last_became_active":"2024-04-26T04:39:56.300067+0000","last_became_peered":"2024-04-26T04:39:56.300067+0000","last_unstale":"2024-04-26T04:59:44.891292+0000","last_undegraded":"2024-04-26T04:59:44.891292+0000","last_fullsized":"2024-04-26T04:59:44.891292+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:53:28.753150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894634+0000","last_change":"2024-04-26T04:59:09.197393+0000","last_active":"2024-04-26T04:59:44.894634+0000","last_peered":"2024-04-26T04:59:44.894634+0000","last_clean":"2024-04-26T04:59:44.894634+0000","last_became_active":"2024-04-26T04:59:09.197059+0000","last_became_peered":"2024-04-26T04:59:09.197059+0000","last_unstale":"2024-04-26T04:59:44.894634+0000","last_undegraded":"2024-04-26T04:59:44.894634+0000","last_fullsized":"2024-04-26T04:59:44.894634+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:52:39.077213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897986+0000","last_change":"2024-04-26T04:59:45.897986+0000","last_active":"2024-04-26T04:59:45.897986+0000","last_peered":"2024-04-26T04:59:45.897986+0000","last_clean":"2024-04-26T04:59:45.897986+0000","last_became_active":"2024-04-26T04:59:45.897623+0000","last_became_peered":"2024-04-26T04:59:45.897623+0000","last_unstale":"2024-04-26T04:59:45.897986+0000","last_undegraded":"2024-04-26T04:59:45.897986+0000","last_fullsized":"2024-04-26T04:59:45.897986+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:49:05.163905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887984+0000","last_change":"2024-04-26T04:54:53.658384+0000","last_active":"2024-04-26T04:59:45.887984+0000","last_peered":"2024-04-26T04:59:45.887984+0000","last_clean":"2024-04-26T04:59:45.887984+0000","last_became_active":"2024-04-26T04:54:53.658051+0000","last_became_peered":"2024-04-26T04:54:53.658051+0000","last_unstale":"2024-04-26T04:59:45.887984+0000","last_undegraded":"2024-04-26T04:59:45.887984+0000","last_fullsized":"2024-04-26T04:59:45.887984+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:57:25.649226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":67,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.899907+0000","last_change":"2024-04-26T04:59:45.899907+0000","last_active":"2024-04-26T04:59:45.899907+0000","last_peered":"2024-04-26T04:59:45.899907+0000","last_clean":"2024-04-26T04:59:45.899907+0000","last_became_active":"2024-04-26T04:59:45.899536+0000","last_became_peered":"2024-04-26T04:59:45.899536+0000","last_unstale":"2024-04-26T04:59:45.899907+0000","last_undegraded":"2024-04-26T04:59:45.899907+0000","last_fullsized":"2024-04-26T04:59:45.899907+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:26:58.697020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":91,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900947+0000","last_change":"2024-04-26T04:59:45.900947+0000","last_active":"2024-04-26T04:59:45.900947+0000","last_peered":"2024-04-26T04:59:45.900947+0000","last_clean":"2024-04-26T04:59:45.900947+0000","last_became_active":"2024-04-26T04:59:45.900530+0000","last_became_peered":"2024-04-26T04:59:45.900530+0000","last_unstale":"2024-04-26T04:59:45.900947+0000","last_undegraded":"2024-04-26T04:59:45.900947+0000","last_fullsized":"2024-04-26T04:59:45.900947+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:09.322330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894471+0000","last_change":"2024-04-26T04:54:53.661721+0000","last_active":"2024-04-26T04:59:44.894471+0000","last_peered":"2024-04-26T04:59:44.894471+0000","last_clean":"2024-04-26T04:59:44.894471+0000","last_became_active":"2024-04-26T04:54:53.661321+0000","last_became_peered":"2024-04-26T04:54:53.661321+0000","last_unstale":"2024-04-26T04:59:44.894471+0000","last_undegraded":"2024-04-26T04:59:44.894471+0000","last_fullsized":"2024-04-26T04:59:44.894471+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:06:13.819647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894868+0000","last_change":"2024-04-26T04:59:09.198293+0000","last_active":"2024-04-26T04:59:44.894868+0000","last_peered":"2024-04-26T04:59:44.894868+0000","last_clean":"2024-04-26T04:59:44.894868+0000","last_became_active":"2024-04-26T04:59:09.197627+0000","last_became_peered":"2024-04-26T04:59:09.197627+0000","last_unstale":"2024-04-26T04:59:44.894868+0000","last_undegraded":"2024-04-26T04:59:44.894868+0000","last_fullsized":"2024-04-26T04:59:44.894868+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:38:51.845227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.899459+0000","last_change":"2024-04-26T04:59:09.198761+0000","last_active":"2024-04-26T04:59:44.899459+0000","last_peered":"2024-04-26T04:59:44.899459+0000","last_clean":"2024-04-26T04:59:44.899459+0000","last_became_active":"2024-04-26T04:59:09.197730+0000","last_became_peered":"2024-04-26T04:59:09.197730+0000","last_unstale":"2024-04-26T04:59:44.899459+0000","last_undegraded":"2024-04-26T04:59:44.899459+0000","last_fullsized":"2024-04-26T04:59:44.899459+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:47:14.999923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.896104+0000","last_change":"2024-04-26T04:54:53.656773+0000","last_active":"2024-04-26T04:59:44.896104+0000","last_peered":"2024-04-26T04:59:44.896104+0000","last_clean":"2024-04-26T04:59:44.896104+0000","last_became_active":"2024-04-26T04:54:53.655013+0000","last_became_peered":"2024-04-26T04:54:53.655013+0000","last_unstale":"2024-04-26T04:59:44.896104+0000","last_undegraded":"2024-04-26T04:59:44.896104+0000","last_fullsized":"2024-04-26T04:59:44.896104+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:36:08.749702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906067+0000","last_change":"2024-04-26T04:59:45.906067+0000","last_active":"2024-04-26T04:59:45.906067+0000","last_peered":"2024-04-26T04:59:45.906067+0000","last_clean":"2024-04-26T04:59:45.906067+0000","last_became_active":"2024-04-26T04:59:45.902715+0000","last_became_peered":"2024-04-26T04:59:45.902715+0000","last_unstale":"2024-04-26T04:59:45.906067+0000","last_undegraded":"2024-04-26T04:59:45.906067+0000","last_fullsized":"2024-04-26T04:59:45.906067+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:50:51.953033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896147+0000","last_change":"2024-04-26T04:59:45.896147+0000","last_active":"2024-04-26T04:59:45.896147+0000","last_peered":"2024-04-26T04:59:45.896147+0000","last_clean":"2024-04-26T04:59:45.896147+0000","last_became_active":"2024-04-26T04:59:45.895858+0000","last_became_peered":"2024-04-26T04:59:45.895858+0000","last_unstale":"2024-04-26T04:59:45.896147+0000","last_undegraded":"2024-04-26T04:59:45.896147+0000","last_fullsized":"2024-04-26T04:59:45.896147+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:31:33.505661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887986+0000","last_change":"2024-04-26T04:54:53.658222+0000","last_active":"2024-04-26T04:59:45.887986+0000","last_peered":"2024-04-26T04:59:45.887986+0000","last_clean":"2024-04-26T04:59:45.887986+0000","last_became_active":"2024-04-26T04:54:53.656150+0000","last_became_peered":"2024-04-26T04:54:53.656150+0000","last_unstale":"2024-04-26T04:59:45.887986+0000","last_undegraded":"2024-04-26T04:59:45.887986+0000","last_fullsized":"2024-04-26T04:59:45.887986+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:22:25.670211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.898008+0000","last_change":"2024-04-26T04:59:45.898008+0000","last_active":"2024-04-26T04:59:45.898008+0000","last_peered":"2024-04-26T04:59:45.898008+0000","last_clean":"2024-04-26T04:59:45.898008+0000","last_became_active":"2024-04-26T04:59:45.897695+0000","last_became_peered":"2024-04-26T04:59:45.897695+0000","last_unstale":"2024-04-26T04:59:45.898008+0000","last_undegraded":"2024-04-26T04:59:45.898008+0000","last_fullsized":"2024-04-26T04:59:45.898008+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:30:40.519787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887575+0000","last_change":"2024-04-26T04:54:53.659780+0000","last_active":"2024-04-26T04:59:45.887575+0000","last_peered":"2024-04-26T04:59:45.887575+0000","last_clean":"2024-04-26T04:59:45.887575+0000","last_became_active":"2024-04-26T04:54:53.659228+0000","last_became_peered":"2024-04-26T04:54:53.659228+0000","last_unstale":"2024-04-26T04:59:45.887575+0000","last_undegraded":"2024-04-26T04:59:45.887575+0000","last_fullsized":"2024-04-26T04:59:45.887575+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:14:12.331200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889555+0000","last_change":"2024-04-26T04:54:53.655029+0000","last_active":"2024-04-26T04:59:44.889555+0000","last_peered":"2024-04-26T04:59:44.889555+0000","last_clean":"2024-04-26T04:59:44.889555+0000","last_became_active":"2024-04-26T04:54:53.654706+0000","last_became_peered":"2024-04-26T04:54:53.654706+0000","last_unstale":"2024-04-26T04:59:44.889555+0000","last_undegraded":"2024-04-26T04:59:44.889555+0000","last_fullsized":"2024-04-26T04:59:44.889555+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:19:18.843557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.898845+0000","last_change":"2024-04-26T04:51:40.698560+0000","last_active":"2024-04-26T04:59:44.898845+0000","last_peered":"2024-04-26T04:59:44.898845+0000","last_clean":"2024-04-26T04:59:44.898845+0000","last_became_active":"2024-04-26T04:51:40.698168+0000","last_became_peered":"2024-04-26T04:51:40.698168+0000","last_unstale":"2024-04-26T04:59:44.898845+0000","last_undegraded":"2024-04-26T04:59:44.898845+0000","last_fullsized":"2024-04-26T04:59:44.898845+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:13:49.122322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896060+0000","last_change":"2024-04-26T04:59:45.896060+0000","last_active":"2024-04-26T04:59:45.896060+0000","last_peered":"2024-04-26T04:59:45.896060+0000","last_clean":"2024-04-26T04:59:45.896060+0000","last_became_active":"2024-04-26T04:59:45.895737+0000","last_became_peered":"2024-04-26T04:59:45.895737+0000","last_unstale":"2024-04-26T04:59:45.896060+0000","last_undegraded":"2024-04-26T04:59:45.896060+0000","last_fullsized":"2024-04-26T04:59:45.896060+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:41.274761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":88,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.893896+0000","last_change":"2024-04-26T04:59:45.893896+0000","last_active":"2024-04-26T04:59:45.893896+0000","last_peered":"2024-04-26T04:59:45.893896+0000","last_clean":"2024-04-26T04:59:45.893896+0000","last_became_active":"2024-04-26T04:59:45.893525+0000","last_became_peered":"2024-04-26T04:59:45.893525+0000","last_unstale":"2024-04-26T04:59:45.893896+0000","last_undegraded":"2024-04-26T04:59:45.893896+0000","last_fullsized":"2024-04-26T04:59:45.893896+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:55:24.620601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888119+0000","last_change":"2024-04-26T04:59:09.197177+0000","last_active":"2024-04-26T04:59:45.888119+0000","last_peered":"2024-04-26T04:59:45.888119+0000","last_clean":"2024-04-26T04:59:45.888119+0000","last_became_active":"2024-04-26T04:59:09.196844+0000","last_became_peered":"2024-04-26T04:59:09.196844+0000","last_unstale":"2024-04-26T04:59:45.888119+0000","last_undegraded":"2024-04-26T04:59:45.888119+0000","last_fullsized":"2024-04-26T04:59:45.888119+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:09:21.125512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888619+0000","last_change":"2024-04-26T04:54:53.658857+0000","last_active":"2024-04-26T04:59:45.888619+0000","last_peered":"2024-04-26T04:59:45.888619+0000","last_clean":"2024-04-26T04:59:45.888619+0000","last_became_active":"2024-04-26T04:54:53.658476+0000","last_became_peered":"2024-04-26T04:54:53.658476+0000","last_unstale":"2024-04-26T04:59:45.888619+0000","last_undegraded":"2024-04-26T04:59:45.888619+0000","last_fullsized":"2024-04-26T04:59:45.888619+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:45:09.211999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889480+0000","last_change":"2024-04-26T04:59:09.202553+0000","last_active":"2024-04-26T04:59:44.889480+0000","last_peered":"2024-04-26T04:59:44.889480+0000","last_clean":"2024-04-26T04:59:44.889480+0000","last_became_active":"2024-04-26T04:59:09.202205+0000","last_became_peered":"2024-04-26T04:59:09.202205+0000","last_unstale":"2024-04-26T04:59:44.889480+0000","last_undegraded":"2024-04-26T04:59:44.889480+0000","last_fullsized":"2024-04-26T04:59:44.889480+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:12:10.109522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":473,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.892164+0000","last_change":"2024-04-26T04:51:40.691091+0000","last_active":"2024-04-26T04:59:44.892164+0000","last_peered":"2024-04-26T04:59:44.892164+0000","last_clean":"2024-04-26T04:59:44.892164+0000","last_became_active":"2024-04-26T04:51:40.688992+0000","last_became_peered":"2024-04-26T04:51:40.688992+0000","last_unstale":"2024-04-26T04:59:44.892164+0000","last_undegraded":"2024-04-26T04:59:44.892164+0000","last_fullsized":"2024-04-26T04:59:44.892164+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_clean_scrub_stamp":"2024-04-26T04:36:10.749042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:52:04.890523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903539+0000","last_change":"2024-04-26T04:59:45.903539+0000","last_active":"2024-04-26T04:59:45.903539+0000","last_peered":"2024-04-26T04:59:45.903539+0000","last_clean":"2024-04-26T04:59:45.903539+0000","last_became_active":"2024-04-26T04:59:45.902514+0000","last_became_peered":"2024-04-26T04:59:45.902514+0000","last_unstale":"2024-04-26T04:59:45.903539+0000","last_undegraded":"2024-04-26T04:59:45.903539+0000","last_fullsized":"2024-04-26T04:59:45.903539+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:11:49.800234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":74,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895226+0000","last_change":"2024-04-26T04:39:56.923636+0000","last_active":"2024-04-26T04:59:44.895226+0000","last_peered":"2024-04-26T04:59:44.895226+0000","last_clean":"2024-04-26T04:59:44.895226+0000","last_became_active":"2024-04-26T04:39:56.923324+0000","last_became_peered":"2024-04-26T04:39:56.923324+0000","last_unstale":"2024-04-26T04:59:44.895226+0000","last_undegraded":"2024-04-26T04:59:44.895226+0000","last_fullsized":"2024-04-26T04:59:44.895226+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:05:54.903060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891469+0000","last_change":"2024-04-26T04:54:53.656591+0000","last_active":"2024-04-26T04:59:44.891469+0000","last_peered":"2024-04-26T04:59:44.891469+0000","last_clean":"2024-04-26T04:59:44.891469+0000","last_became_active":"2024-04-26T04:54:53.656008+0000","last_became_peered":"2024-04-26T04:54:53.656008+0000","last_unstale":"2024-04-26T04:59:44.891469+0000","last_undegraded":"2024-04-26T04:59:44.891469+0000","last_fullsized":"2024-04-26T04:59:44.891469+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:51:21.167848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906776+0000","last_change":"2024-04-26T04:59:45.905687+0000","last_active":"2024-04-26T04:59:45.906776+0000","last_peered":"2024-04-26T04:59:45.906776+0000","last_clean":"2024-04-26T04:59:45.906776+0000","last_became_active":"2024-04-26T04:59:45.905267+0000","last_became_peered":"2024-04-26T04:59:45.905267+0000","last_unstale":"2024-04-26T04:59:45.906776+0000","last_undegraded":"2024-04-26T04:59:45.906776+0000","last_fullsized":"2024-04-26T04:59:45.906776+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:49:14.504104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.890903+0000","last_change":"2024-04-26T04:59:09.197935+0000","last_active":"2024-04-26T04:59:45.890903+0000","last_peered":"2024-04-26T04:59:45.890903+0000","last_clean":"2024-04-26T04:59:45.890903+0000","last_became_active":"2024-04-26T04:59:09.197571+0000","last_became_peered":"2024-04-26T04:59:09.197571+0000","last_unstale":"2024-04-26T04:59:45.890903+0000","last_undegraded":"2024-04-26T04:59:45.890903+0000","last_fullsized":"2024-04-26T04:59:45.890903+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:01:39.998596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891729+0000","last_change":"2024-04-26T04:51:40.781926+0000","last_active":"2024-04-26T04:59:44.891729+0000","last_peered":"2024-04-26T04:59:44.891729+0000","last_clean":"2024-04-26T04:59:44.891729+0000","last_became_active":"2024-04-26T04:51:40.781540+0000","last_became_peered":"2024-04-26T04:51:40.781540+0000","last_unstale":"2024-04-26T04:59:44.891729+0000","last_undegraded":"2024-04-26T04:59:44.891729+0000","last_fullsized":"2024-04-26T04:59:44.891729+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:43:44.834128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888119+0000","last_change":"2024-04-26T04:59:09.199954+0000","last_active":"2024-04-26T04:59:45.888119+0000","last_peered":"2024-04-26T04:59:45.888119+0000","last_clean":"2024-04-26T04:59:45.888119+0000","last_became_active":"2024-04-26T04:59:09.199312+0000","last_became_peered":"2024-04-26T04:59:09.199312+0000","last_unstale":"2024-04-26T04:59:45.888119+0000","last_undegraded":"2024-04-26T04:59:45.888119+0000","last_fullsized":"2024-04-26T04:59:45.888119+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:22:17.334848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":106,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887709+0000","last_change":"2024-04-26T04:51:40.708519+0000","last_active":"2024-04-26T04:59:45.887709+0000","last_peered":"2024-04-26T04:59:45.887709+0000","last_clean":"2024-04-26T04:59:45.887709+0000","last_became_active":"2024-04-26T04:51:40.702193+0000","last_became_peered":"2024-04-26T04:51:40.702193+0000","last_unstale":"2024-04-26T04:59:45.887709+0000","last_undegraded":"2024-04-26T04:59:45.887709+0000","last_fullsized":"2024-04-26T04:59:45.887709+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:53:20.759824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":67,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906445+0000","last_change":"2024-04-26T04:59:45.906445+0000","last_active":"2024-04-26T04:59:45.906445+0000","last_peered":"2024-04-26T04:59:45.906445+0000","last_clean":"2024-04-26T04:59:45.906445+0000","last_became_active":"2024-04-26T04:59:45.906096+0000","last_became_peered":"2024-04-26T04:59:45.906096+0000","last_unstale":"2024-04-26T04:59:45.906445+0000","last_undegraded":"2024-04-26T04:59:45.906445+0000","last_fullsized":"2024-04-26T04:59:45.906445+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:18:15.331535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887854+0000","last_change":"2024-04-26T04:54:53.660752+0000","last_active":"2024-04-26T04:59:45.887854+0000","last_peered":"2024-04-26T04:59:45.887854+0000","last_clean":"2024-04-26T04:59:45.887854+0000","last_became_active":"2024-04-26T04:54:53.659802+0000","last_became_peered":"2024-04-26T04:54:53.659802+0000","last_unstale":"2024-04-26T04:59:45.887854+0000","last_undegraded":"2024-04-26T04:59:45.887854+0000","last_fullsized":"2024-04-26T04:59:45.887854+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:34:12.694764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.902730+0000","last_change":"2024-04-26T04:59:45.902730+0000","last_active":"2024-04-26T04:59:45.902730+0000","last_peered":"2024-04-26T04:59:45.902730+0000","last_clean":"2024-04-26T04:59:45.902730+0000","last_became_active":"2024-04-26T04:59:45.901445+0000","last_became_peered":"2024-04-26T04:59:45.901445+0000","last_unstale":"2024-04-26T04:59:45.902730+0000","last_undegraded":"2024-04-26T04:59:45.902730+0000","last_fullsized":"2024-04-26T04:59:45.902730+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:53:56.765790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894658+0000","last_change":"2024-04-26T04:59:09.198231+0000","last_active":"2024-04-26T04:59:44.894658+0000","last_peered":"2024-04-26T04:59:44.894658+0000","last_clean":"2024-04-26T04:59:44.894658+0000","last_became_active":"2024-04-26T04:59:09.197829+0000","last_became_peered":"2024-04-26T04:59:09.197829+0000","last_unstale":"2024-04-26T04:59:44.894658+0000","last_undegraded":"2024-04-26T04:59:44.894658+0000","last_fullsized":"2024-04-26T04:59:44.894658+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:13:00.464779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.890371+0000","last_change":"2024-04-26T04:54:53.655426+0000","last_active":"2024-04-26T04:59:45.890371+0000","last_peered":"2024-04-26T04:59:45.890371+0000","last_clean":"2024-04-26T04:59:45.890371+0000","last_became_active":"2024-04-26T04:54:53.653483+0000","last_became_peered":"2024-04-26T04:54:53.653483+0000","last_unstale":"2024-04-26T04:59:45.890371+0000","last_undegraded":"2024-04-26T04:59:45.890371+0000","last_fullsized":"2024-04-26T04:59:45.890371+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:13:21.331869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.899549+0000","last_change":"2024-04-26T04:39:56.315667+0000","last_active":"2024-04-26T04:59:44.899549+0000","last_peered":"2024-04-26T04:59:44.899549+0000","last_clean":"2024-04-26T04:59:44.899549+0000","last_became_active":"2024-04-26T04:39:56.315353+0000","last_became_peered":"2024-04-26T04:39:56.315353+0000","last_unstale":"2024-04-26T04:59:44.899549+0000","last_undegraded":"2024-04-26T04:59:44.899549+0000","last_fullsized":"2024-04-26T04:59:44.899549+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:45:00.659197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888041+0000","last_change":"2024-04-26T04:51:40.707690+0000","last_active":"2024-04-26T04:59:45.888041+0000","last_peered":"2024-04-26T04:59:45.888041+0000","last_clean":"2024-04-26T04:59:45.888041+0000","last_became_active":"2024-04-26T04:51:40.700784+0000","last_became_peered":"2024-04-26T04:51:40.700784+0000","last_unstale":"2024-04-26T04:59:45.888041+0000","last_undegraded":"2024-04-26T04:59:45.888041+0000","last_fullsized":"2024-04-26T04:59:45.888041+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:57:12.082663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896140+0000","last_change":"2024-04-26T04:59:45.896140+0000","last_active":"2024-04-26T04:59:45.896140+0000","last_peered":"2024-04-26T04:59:45.896140+0000","last_clean":"2024-04-26T04:59:45.896140+0000","last_became_active":"2024-04-26T04:59:45.895738+0000","last_became_peered":"2024-04-26T04:59:45.895738+0000","last_unstale":"2024-04-26T04:59:45.896140+0000","last_undegraded":"2024-04-26T04:59:45.896140+0000","last_fullsized":"2024-04-26T04:59:45.896140+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:24:24.596759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.890835+0000","last_change":"2024-04-26T04:54:53.654261+0000","last_active":"2024-04-26T04:59:44.890835+0000","last_peered":"2024-04-26T04:59:44.890835+0000","last_clean":"2024-04-26T04:59:44.890835+0000","last_became_active":"2024-04-26T04:54:53.653828+0000","last_became_peered":"2024-04-26T04:54:53.653828+0000","last_unstale":"2024-04-26T04:59:44.890835+0000","last_undegraded":"2024-04-26T04:59:44.890835+0000","last_fullsized":"2024-04-26T04:59:44.890835+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:04.603924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900628+0000","last_change":"2024-04-26T04:59:45.900628+0000","last_active":"2024-04-26T04:59:45.900628+0000","last_peered":"2024-04-26T04:59:45.900628+0000","last_clean":"2024-04-26T04:59:45.900628+0000","last_became_active":"2024-04-26T04:59:45.900288+0000","last_became_peered":"2024-04-26T04:59:45.900288+0000","last_unstale":"2024-04-26T04:59:45.900628+0000","last_undegraded":"2024-04-26T04:59:45.900628+0000","last_fullsized":"2024-04-26T04:59:45.900628+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:25:45.956189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":77,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.892441+0000","last_change":"2024-04-26T04:39:56.299459+0000","last_active":"2024-04-26T04:59:44.892441+0000","last_peered":"2024-04-26T04:59:44.892441+0000","last_clean":"2024-04-26T04:59:44.892441+0000","last_became_active":"2024-04-26T04:39:56.299180+0000","last_became_peered":"2024-04-26T04:39:56.299180+0000","last_unstale":"2024-04-26T04:59:44.892441+0000","last_undegraded":"2024-04-26T04:59:44.892441+0000","last_fullsized":"2024-04-26T04:59:44.892441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:51:01.512924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900459+0000","last_change":"2024-04-26T04:59:45.900459+0000","last_active":"2024-04-26T04:59:45.900459+0000","last_peered":"2024-04-26T04:59:45.900459+0000","last_clean":"2024-04-26T04:59:45.900459+0000","last_became_active":"2024-04-26T04:59:45.900067+0000","last_became_peered":"2024-04-26T04:59:45.900067+0000","last_unstale":"2024-04-26T04:59:45.900459+0000","last_undegraded":"2024-04-26T04:59:45.900459+0000","last_fullsized":"2024-04-26T04:59:45.900459+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:28:11.952886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897624+0000","last_change":"2024-04-26T04:59:45.897624+0000","last_active":"2024-04-26T04:59:45.897624+0000","last_peered":"2024-04-26T04:59:45.897624+0000","last_clean":"2024-04-26T04:59:45.897624+0000","last_became_active":"2024-04-26T04:59:45.897151+0000","last_became_peered":"2024-04-26T04:59:45.897151+0000","last_unstale":"2024-04-26T04:59:45.897624+0000","last_undegraded":"2024-04-26T04:59:45.897624+0000","last_fullsized":"2024-04-26T04:59:45.897624+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:14:26.180049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897199+0000","last_change":"2024-04-26T04:59:45.897199+0000","last_active":"2024-04-26T04:59:45.897199+0000","last_peered":"2024-04-26T04:59:45.897199+0000","last_clean":"2024-04-26T04:59:45.897199+0000","last_became_active":"2024-04-26T04:59:45.896884+0000","last_became_peered":"2024-04-26T04:59:45.896884+0000","last_unstale":"2024-04-26T04:59:45.897199+0000","last_undegraded":"2024-04-26T04:59:45.897199+0000","last_fullsized":"2024-04-26T04:59:45.897199+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:55:45.414733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":119,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888649+0000","last_change":"2024-04-26T04:59:09.199450+0000","last_active":"2024-04-26T04:59:45.888649+0000","last_peered":"2024-04-26T04:59:45.888649+0000","last_clean":"2024-04-26T04:59:45.888649+0000","last_became_active":"2024-04-26T04:59:09.199021+0000","last_became_peered":"2024-04-26T04:59:09.199021+0000","last_unstale":"2024-04-26T04:59:45.888649+0000","last_undegraded":"2024-04-26T04:59:45.888649+0000","last_fullsized":"2024-04-26T04:59:45.888649+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:56:38.218588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.893785+0000","last_change":"2024-04-26T04:54:53.662687+0000","last_active":"2024-04-26T04:59:44.893785+0000","last_peered":"2024-04-26T04:59:44.893785+0000","last_clean":"2024-04-26T04:59:44.893785+0000","last_became_active":"2024-04-26T04:54:53.662153+0000","last_became_peered":"2024-04-26T04:54:53.662153+0000","last_unstale":"2024-04-26T04:59:44.893785+0000","last_undegraded":"2024-04-26T04:59:44.893785+0000","last_fullsized":"2024-04-26T04:59:44.893785+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:49:36.123381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13661,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201552,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307660,"kb_used_data":2112,"kb_used_omap":1,"kb_used_meta":305534,"kb_avail":93437492,"statfs":{"total":95995035648,"available":95679991808,"internally_reserved":0,"allocated":2162688,"data_stored":1429488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312867160},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463189,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311752,"kb_used_data":2108,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433400,"statfs":{"total":95995035648,"available":95675801600,"internally_reserved":0,"allocated":2158592,"data_stored":1429888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":113,"seq":485331304520,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":203576,"kb_used_data":1508,"kb_used_omap":5,"kb_used_meta":202042,"kb_avail":93541576,"statfs":{"total":95995035648,"available":95786573824,"internally_reserved":0,"allocated":1544192,"data_stored":834453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":206891287},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":91,"seq":390842024046,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":257988,"kb_used_data":2100,"kb_used_omap":5,"kb_used_meta":255866,"kb_avail":93487164,"statfs":{"total":95995035648,"available":95730855936,"internally_reserved":0,"allocated":2150400,"data_stored":1425694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6052,"internal_metadata":262006876},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879317,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":52412,"kb_used_data":1520,"kb_used_omap":6,"kb_used_meta":50873,"kb_avail":93692740,"statfs":{"total":95995035648,"available":95941365760,"internally_reserved":0,"allocated":1556480,"data_stored":835338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6732,"internal_metadata":52094388},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":131,"seq":562640715789,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77952,"kb_used_data":1516,"kb_used_omap":4,"kb_used_meta":76411,"kb_avail":93667200,"statfs":{"total":95995035648,"available":95915212800,"internally_reserved":0,"allocated":1552384,"data_stored":834816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4639,"internal_metadata":78245345},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986463,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599564,"kb_used_data":1532,"kb_used_omap":1,"kb_used_meta":598014,"kb_avail":93145588,"statfs":{"total":95995035648,"available":95381082112,"internally_reserved":0,"allocated":1568768,"data_stored":839331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612366680},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724825,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897208,"kb_used_data":1512,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92847944,"statfs":{"total":95995035648,"available":95076294656,"internally_reserved":0,"allocated":1548288,"data_stored":834750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4348,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2217,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2935,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:00:45.450 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-26T05:00:45.759 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:46 smithi038 ceph-mon[93172]: from='client.35241 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:00:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:46 smithi038 ceph-mon[93172]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:46 smithi038 ceph-mon[96929]: from='client.35241 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:00:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:46 smithi038 ceph-mon[96929]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:46.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:46 smithi055 ceph-mon[86906]: from='client.35241 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:00:46.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:46 smithi055 ceph-mon[86906]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:47.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:00:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:00:46] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:00:47.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:47.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:47.653 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:47.654 INFO:teuthology.orchestra.run.smithi038.stderr:dumped all 2024-04-26T05:00:47.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:00:48.291 INFO:teuthology.orchestra.run.smithi038.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2024-04-26T05:00:47.234691+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2708112,"kb_used_data":13908,"kb_used_omap":29,"kb_used_meta":2694050,"kb_avail":747253104,"statfs":{"total":767960285184,"available":765187178496,"internally_reserved":0,"allocated":14241792,"data_stored":8463758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30104,"internal_metadata":2758707816},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002893"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":110,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.897873+0000","last_change":"2024-04-26T04:59:09.197430+0000","last_active":"2024-04-26T04:59:44.897873+0000","last_peered":"2024-04-26T04:59:44.897873+0000","last_clean":"2024-04-26T04:59:44.897873+0000","last_became_active":"2024-04-26T04:59:09.197082+0000","last_became_peered":"2024-04-26T04:59:09.197082+0000","last_unstale":"2024-04-26T04:59:44.897873+0000","last_undegraded":"2024-04-26T04:59:44.897873+0000","last_fullsized":"2024-04-26T04:59:44.897873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:02:36.345011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903180+0000","last_change":"2024-04-26T04:59:45.903180+0000","last_active":"2024-04-26T04:59:45.903180+0000","last_peered":"2024-04-26T04:59:45.903180+0000","last_clean":"2024-04-26T04:59:45.903180+0000","last_became_active":"2024-04-26T04:59:45.902801+0000","last_became_peered":"2024-04-26T04:59:45.902801+0000","last_unstale":"2024-04-26T04:59:45.903180+0000","last_undegraded":"2024-04-26T04:59:45.903180+0000","last_fullsized":"2024-04-26T04:59:45.903180+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:36.071038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896797+0000","last_change":"2024-04-26T04:59:45.896797+0000","last_active":"2024-04-26T04:59:45.896797+0000","last_peered":"2024-04-26T04:59:45.896797+0000","last_clean":"2024-04-26T04:59:45.896797+0000","last_became_active":"2024-04-26T04:59:45.896456+0000","last_became_peered":"2024-04-26T04:59:45.896456+0000","last_unstale":"2024-04-26T04:59:45.896797+0000","last_undegraded":"2024-04-26T04:59:45.896797+0000","last_fullsized":"2024-04-26T04:59:45.896797+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:23:19.527315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905441+0000","last_change":"2024-04-26T04:59:45.905441+0000","last_active":"2024-04-26T04:59:45.905441+0000","last_peered":"2024-04-26T04:59:45.905441+0000","last_clean":"2024-04-26T04:59:45.905441+0000","last_became_active":"2024-04-26T04:59:45.905077+0000","last_became_peered":"2024-04-26T04:59:45.905077+0000","last_unstale":"2024-04-26T04:59:45.905441+0000","last_undegraded":"2024-04-26T04:59:45.905441+0000","last_fullsized":"2024-04-26T04:59:45.905441+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:01:03.311399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897272+0000","last_change":"2024-04-26T04:59:45.897272+0000","last_active":"2024-04-26T04:59:45.897272+0000","last_peered":"2024-04-26T04:59:45.897272+0000","last_clean":"2024-04-26T04:59:45.897272+0000","last_became_active":"2024-04-26T04:59:45.896838+0000","last_became_peered":"2024-04-26T04:59:45.896838+0000","last_unstale":"2024-04-26T04:59:45.897272+0000","last_undegraded":"2024-04-26T04:59:45.897272+0000","last_fullsized":"2024-04-26T04:59:45.897272+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:00:06.719287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":101,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895326+0000","last_change":"2024-04-26T04:59:09.198697+0000","last_active":"2024-04-26T04:59:44.895326+0000","last_peered":"2024-04-26T04:59:44.895326+0000","last_clean":"2024-04-26T04:59:44.895326+0000","last_became_active":"2024-04-26T04:59:09.198364+0000","last_became_peered":"2024-04-26T04:59:09.198364+0000","last_unstale":"2024-04-26T04:59:44.895326+0000","last_undegraded":"2024-04-26T04:59:44.895326+0000","last_fullsized":"2024-04-26T04:59:44.895326+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:37:52.991031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906493+0000","last_change":"2024-04-26T04:59:45.906493+0000","last_active":"2024-04-26T04:59:45.906493+0000","last_peered":"2024-04-26T04:59:45.906493+0000","last_clean":"2024-04-26T04:59:45.906493+0000","last_became_active":"2024-04-26T04:59:45.906198+0000","last_became_peered":"2024-04-26T04:59:45.906198+0000","last_unstale":"2024-04-26T04:59:45.906493+0000","last_undegraded":"2024-04-26T04:59:45.906493+0000","last_fullsized":"2024-04-26T04:59:45.906493+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:12:50.480125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.898046+0000","last_change":"2024-04-26T04:59:45.898046+0000","last_active":"2024-04-26T04:59:45.898046+0000","last_peered":"2024-04-26T04:59:45.898046+0000","last_clean":"2024-04-26T04:59:45.898046+0000","last_became_active":"2024-04-26T04:59:45.897507+0000","last_became_peered":"2024-04-26T04:59:45.897507+0000","last_unstale":"2024-04-26T04:59:45.898046+0000","last_undegraded":"2024-04-26T04:59:45.898046+0000","last_fullsized":"2024-04-26T04:59:45.898046+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:45:23.980125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889543+0000","last_change":"2024-04-26T04:54:53.653738+0000","last_active":"2024-04-26T04:59:44.889543+0000","last_peered":"2024-04-26T04:59:44.889543+0000","last_clean":"2024-04-26T04:59:44.889543+0000","last_became_active":"2024-04-26T04:54:53.653382+0000","last_became_peered":"2024-04-26T04:54:53.653382+0000","last_unstale":"2024-04-26T04:59:44.889543+0000","last_undegraded":"2024-04-26T04:59:44.889543+0000","last_fullsized":"2024-04-26T04:59:44.889543+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:32:37.322919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903161+0000","last_change":"2024-04-26T04:59:45.903161+0000","last_active":"2024-04-26T04:59:45.903161+0000","last_peered":"2024-04-26T04:59:45.903161+0000","last_clean":"2024-04-26T04:59:45.903161+0000","last_became_active":"2024-04-26T04:59:45.902756+0000","last_became_peered":"2024-04-26T04:59:45.902756+0000","last_unstale":"2024-04-26T04:59:45.903161+0000","last_undegraded":"2024-04-26T04:59:45.903161+0000","last_fullsized":"2024-04-26T04:59:45.903161+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:24:02.116202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":95,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905512+0000","last_change":"2024-04-26T04:59:45.905512+0000","last_active":"2024-04-26T04:59:45.905512+0000","last_peered":"2024-04-26T04:59:45.905512+0000","last_clean":"2024-04-26T04:59:45.905512+0000","last_became_active":"2024-04-26T04:59:45.905117+0000","last_became_peered":"2024-04-26T04:59:45.905117+0000","last_unstale":"2024-04-26T04:59:45.905512+0000","last_undegraded":"2024-04-26T04:59:45.905512+0000","last_fullsized":"2024-04-26T04:59:45.905512+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:38:53.980299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887699+0000","last_change":"2024-04-26T04:59:09.194955+0000","last_active":"2024-04-26T04:59:45.887699+0000","last_peered":"2024-04-26T04:59:45.887699+0000","last_clean":"2024-04-26T04:59:45.887699+0000","last_became_active":"2024-04-26T04:59:09.194612+0000","last_became_peered":"2024-04-26T04:59:09.194612+0000","last_unstale":"2024-04-26T04:59:45.887699+0000","last_undegraded":"2024-04-26T04:59:45.887699+0000","last_fullsized":"2024-04-26T04:59:45.887699+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:57:07.292445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"82'22","reported_seq":97,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895513+0000","last_change":"2024-04-26T04:39:56.300576+0000","last_active":"2024-04-26T04:59:44.895513+0000","last_peered":"2024-04-26T04:59:44.895513+0000","last_clean":"2024-04-26T04:59:44.895513+0000","last_became_active":"2024-04-26T04:39:56.299260+0000","last_became_peered":"2024-04-26T04:39:56.299260+0000","last_unstale":"2024-04-26T04:59:44.895513+0000","last_undegraded":"2024-04-26T04:59:44.895513+0000","last_fullsized":"2024-04-26T04:59:44.895513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:24:26.586898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.898743+0000","last_change":"2024-04-26T04:54:53.663678+0000","last_active":"2024-04-26T04:59:44.898743+0000","last_peered":"2024-04-26T04:59:44.898743+0000","last_clean":"2024-04-26T04:59:44.898743+0000","last_became_active":"2024-04-26T04:54:53.662856+0000","last_became_peered":"2024-04-26T04:54:53.662856+0000","last_unstale":"2024-04-26T04:59:44.898743+0000","last_undegraded":"2024-04-26T04:59:44.898743+0000","last_fullsized":"2024-04-26T04:59:44.898743+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:50:16.153714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.892961+0000","last_change":"2024-04-26T04:59:09.201573+0000","last_active":"2024-04-26T04:59:44.892961+0000","last_peered":"2024-04-26T04:59:44.892961+0000","last_clean":"2024-04-26T04:59:44.892961+0000","last_became_active":"2024-04-26T04:59:09.201166+0000","last_became_peered":"2024-04-26T04:59:09.201166+0000","last_unstale":"2024-04-26T04:59:44.892961+0000","last_undegraded":"2024-04-26T04:59:44.892961+0000","last_fullsized":"2024-04-26T04:59:44.892961+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:39:45.735710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903984+0000","last_change":"2024-04-26T04:59:45.903984+0000","last_active":"2024-04-26T04:59:45.903984+0000","last_peered":"2024-04-26T04:59:45.903984+0000","last_clean":"2024-04-26T04:59:45.903984+0000","last_became_active":"2024-04-26T04:59:45.902811+0000","last_became_peered":"2024-04-26T04:59:45.902811+0000","last_unstale":"2024-04-26T04:59:45.903984+0000","last_undegraded":"2024-04-26T04:59:45.903984+0000","last_fullsized":"2024-04-26T04:59:45.903984+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:57:45.118873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905936+0000","last_change":"2024-04-26T04:59:45.905936+0000","last_active":"2024-04-26T04:59:45.905936+0000","last_peered":"2024-04-26T04:59:45.905936+0000","last_clean":"2024-04-26T04:59:45.905936+0000","last_became_active":"2024-04-26T04:59:45.905272+0000","last_became_peered":"2024-04-26T04:59:45.905272+0000","last_unstale":"2024-04-26T04:59:45.905936+0000","last_undegraded":"2024-04-26T04:59:45.905936+0000","last_fullsized":"2024-04-26T04:59:45.905936+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:39:27.886854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.898668+0000","last_change":"2024-04-26T04:59:45.898668+0000","last_active":"2024-04-26T04:59:45.898668+0000","last_peered":"2024-04-26T04:59:45.898668+0000","last_clean":"2024-04-26T04:59:45.898668+0000","last_became_active":"2024-04-26T04:59:45.898338+0000","last_became_peered":"2024-04-26T04:59:45.898338+0000","last_unstale":"2024-04-26T04:59:45.898668+0000","last_undegraded":"2024-04-26T04:59:45.898668+0000","last_fullsized":"2024-04-26T04:59:45.898668+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:53:39.484941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":75,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891626+0000","last_change":"2024-04-26T04:39:56.302393+0000","last_active":"2024-04-26T04:59:44.891626+0000","last_peered":"2024-04-26T04:59:44.891626+0000","last_clean":"2024-04-26T04:59:44.891626+0000","last_became_active":"2024-04-26T04:39:56.302116+0000","last_became_peered":"2024-04-26T04:39:56.302116+0000","last_unstale":"2024-04-26T04:59:44.891626+0000","last_undegraded":"2024-04-26T04:59:44.891626+0000","last_fullsized":"2024-04-26T04:59:44.891626+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:28:13.475580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905982+0000","last_change":"2024-04-26T04:59:45.905982+0000","last_active":"2024-04-26T04:59:45.905982+0000","last_peered":"2024-04-26T04:59:45.905982+0000","last_clean":"2024-04-26T04:59:45.905982+0000","last_became_active":"2024-04-26T04:59:45.905444+0000","last_became_peered":"2024-04-26T04:59:45.905444+0000","last_unstale":"2024-04-26T04:59:45.905982+0000","last_undegraded":"2024-04-26T04:59:45.905982+0000","last_fullsized":"2024-04-26T04:59:45.905982+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:27:03.195942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":100,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895717+0000","last_change":"2024-04-26T04:51:40.691367+0000","last_active":"2024-04-26T04:59:44.895717+0000","last_peered":"2024-04-26T04:59:44.895717+0000","last_clean":"2024-04-26T04:59:44.895717+0000","last_became_active":"2024-04-26T04:51:40.690968+0000","last_became_peered":"2024-04-26T04:51:40.690968+0000","last_unstale":"2024-04-26T04:59:44.895717+0000","last_undegraded":"2024-04-26T04:59:44.895717+0000","last_fullsized":"2024-04-26T04:59:44.895717+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:51:00.577555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.905593+0000","last_change":"2024-04-26T04:59:45.905593+0000","last_active":"2024-04-26T04:59:45.905593+0000","last_peered":"2024-04-26T04:59:45.905593+0000","last_clean":"2024-04-26T04:59:45.905593+0000","last_became_active":"2024-04-26T04:59:45.904348+0000","last_became_peered":"2024-04-26T04:59:45.904348+0000","last_unstale":"2024-04-26T04:59:45.905593+0000","last_undegraded":"2024-04-26T04:59:45.905593+0000","last_fullsized":"2024-04-26T04:59:45.905593+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:00:33.450842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":93,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.893228+0000","last_change":"2024-04-26T04:51:40.688923+0000","last_active":"2024-04-26T04:59:44.893228+0000","last_peered":"2024-04-26T04:59:44.893228+0000","last_clean":"2024-04-26T04:59:44.893228+0000","last_became_active":"2024-04-26T04:51:40.688072+0000","last_became_peered":"2024-04-26T04:51:40.688072+0000","last_unstale":"2024-04-26T04:59:44.893228+0000","last_undegraded":"2024-04-26T04:59:44.893228+0000","last_fullsized":"2024-04-26T04:59:44.893228+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:35:37.600512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889272+0000","last_change":"2024-04-26T04:39:56.925632+0000","last_active":"2024-04-26T04:59:44.889272+0000","last_peered":"2024-04-26T04:59:44.889272+0000","last_clean":"2024-04-26T04:59:44.889272+0000","last_became_active":"2024-04-26T04:39:56.925327+0000","last_became_peered":"2024-04-26T04:39:56.925327+0000","last_unstale":"2024-04-26T04:59:44.889272+0000","last_undegraded":"2024-04-26T04:59:44.889272+0000","last_fullsized":"2024-04-26T04:59:44.889272+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:29:59.594965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.893141+0000","last_change":"2024-04-26T04:54:53.654360+0000","last_active":"2024-04-26T04:59:44.893141+0000","last_peered":"2024-04-26T04:59:44.893141+0000","last_clean":"2024-04-26T04:59:44.893141+0000","last_became_active":"2024-04-26T04:54:53.653748+0000","last_became_peered":"2024-04-26T04:54:53.653748+0000","last_unstale":"2024-04-26T04:59:44.893141+0000","last_undegraded":"2024-04-26T04:59:44.893141+0000","last_fullsized":"2024-04-26T04:59:44.893141+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:32:03.616453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900765+0000","last_change":"2024-04-26T04:59:45.900765+0000","last_active":"2024-04-26T04:59:45.900765+0000","last_peered":"2024-04-26T04:59:45.900765+0000","last_clean":"2024-04-26T04:59:45.900765+0000","last_became_active":"2024-04-26T04:59:45.900356+0000","last_became_peered":"2024-04-26T04:59:45.900356+0000","last_unstale":"2024-04-26T04:59:45.900765+0000","last_undegraded":"2024-04-26T04:59:45.900765+0000","last_fullsized":"2024-04-26T04:59:45.900765+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:35:37.891548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":76,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891292+0000","last_change":"2024-04-26T04:39:56.300360+0000","last_active":"2024-04-26T04:59:44.891292+0000","last_peered":"2024-04-26T04:59:44.891292+0000","last_clean":"2024-04-26T04:59:44.891292+0000","last_became_active":"2024-04-26T04:39:56.300067+0000","last_became_peered":"2024-04-26T04:39:56.300067+0000","last_unstale":"2024-04-26T04:59:44.891292+0000","last_undegraded":"2024-04-26T04:59:44.891292+0000","last_fullsized":"2024-04-26T04:59:44.891292+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:53:28.753150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894634+0000","last_change":"2024-04-26T04:59:09.197393+0000","last_active":"2024-04-26T04:59:44.894634+0000","last_peered":"2024-04-26T04:59:44.894634+0000","last_clean":"2024-04-26T04:59:44.894634+0000","last_became_active":"2024-04-26T04:59:09.197059+0000","last_became_peered":"2024-04-26T04:59:09.197059+0000","last_unstale":"2024-04-26T04:59:44.894634+0000","last_undegraded":"2024-04-26T04:59:44.894634+0000","last_fullsized":"2024-04-26T04:59:44.894634+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:52:39.077213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897986+0000","last_change":"2024-04-26T04:59:45.897986+0000","last_active":"2024-04-26T04:59:45.897986+0000","last_peered":"2024-04-26T04:59:45.897986+0000","last_clean":"2024-04-26T04:59:45.897986+0000","last_became_active":"2024-04-26T04:59:45.897623+0000","last_became_peered":"2024-04-26T04:59:45.897623+0000","last_unstale":"2024-04-26T04:59:45.897986+0000","last_undegraded":"2024-04-26T04:59:45.897986+0000","last_fullsized":"2024-04-26T04:59:45.897986+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:49:05.163905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887984+0000","last_change":"2024-04-26T04:54:53.658384+0000","last_active":"2024-04-26T04:59:45.887984+0000","last_peered":"2024-04-26T04:59:45.887984+0000","last_clean":"2024-04-26T04:59:45.887984+0000","last_became_active":"2024-04-26T04:54:53.658051+0000","last_became_peered":"2024-04-26T04:54:53.658051+0000","last_unstale":"2024-04-26T04:59:45.887984+0000","last_undegraded":"2024-04-26T04:59:45.887984+0000","last_fullsized":"2024-04-26T04:59:45.887984+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:57:25.649226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":67,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.899907+0000","last_change":"2024-04-26T04:59:45.899907+0000","last_active":"2024-04-26T04:59:45.899907+0000","last_peered":"2024-04-26T04:59:45.899907+0000","last_clean":"2024-04-26T04:59:45.899907+0000","last_became_active":"2024-04-26T04:59:45.899536+0000","last_became_peered":"2024-04-26T04:59:45.899536+0000","last_unstale":"2024-04-26T04:59:45.899907+0000","last_undegraded":"2024-04-26T04:59:45.899907+0000","last_fullsized":"2024-04-26T04:59:45.899907+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:26:58.697020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":91,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900947+0000","last_change":"2024-04-26T04:59:45.900947+0000","last_active":"2024-04-26T04:59:45.900947+0000","last_peered":"2024-04-26T04:59:45.900947+0000","last_clean":"2024-04-26T04:59:45.900947+0000","last_became_active":"2024-04-26T04:59:45.900530+0000","last_became_peered":"2024-04-26T04:59:45.900530+0000","last_unstale":"2024-04-26T04:59:45.900947+0000","last_undegraded":"2024-04-26T04:59:45.900947+0000","last_fullsized":"2024-04-26T04:59:45.900947+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:11:09.322330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894471+0000","last_change":"2024-04-26T04:54:53.661721+0000","last_active":"2024-04-26T04:59:44.894471+0000","last_peered":"2024-04-26T04:59:44.894471+0000","last_clean":"2024-04-26T04:59:44.894471+0000","last_became_active":"2024-04-26T04:54:53.661321+0000","last_became_peered":"2024-04-26T04:54:53.661321+0000","last_unstale":"2024-04-26T04:59:44.894471+0000","last_undegraded":"2024-04-26T04:59:44.894471+0000","last_fullsized":"2024-04-26T04:59:44.894471+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:06:13.819647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894868+0000","last_change":"2024-04-26T04:59:09.198293+0000","last_active":"2024-04-26T04:59:44.894868+0000","last_peered":"2024-04-26T04:59:44.894868+0000","last_clean":"2024-04-26T04:59:44.894868+0000","last_became_active":"2024-04-26T04:59:09.197627+0000","last_became_peered":"2024-04-26T04:59:09.197627+0000","last_unstale":"2024-04-26T04:59:44.894868+0000","last_undegraded":"2024-04-26T04:59:44.894868+0000","last_fullsized":"2024-04-26T04:59:44.894868+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:38:51.845227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.899459+0000","last_change":"2024-04-26T04:59:09.198761+0000","last_active":"2024-04-26T04:59:44.899459+0000","last_peered":"2024-04-26T04:59:44.899459+0000","last_clean":"2024-04-26T04:59:44.899459+0000","last_became_active":"2024-04-26T04:59:09.197730+0000","last_became_peered":"2024-04-26T04:59:09.197730+0000","last_unstale":"2024-04-26T04:59:44.899459+0000","last_undegraded":"2024-04-26T04:59:44.899459+0000","last_fullsized":"2024-04-26T04:59:44.899459+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:47:14.999923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.896104+0000","last_change":"2024-04-26T04:54:53.656773+0000","last_active":"2024-04-26T04:59:44.896104+0000","last_peered":"2024-04-26T04:59:44.896104+0000","last_clean":"2024-04-26T04:59:44.896104+0000","last_became_active":"2024-04-26T04:54:53.655013+0000","last_became_peered":"2024-04-26T04:54:53.655013+0000","last_unstale":"2024-04-26T04:59:44.896104+0000","last_undegraded":"2024-04-26T04:59:44.896104+0000","last_fullsized":"2024-04-26T04:59:44.896104+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:36:08.749702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906067+0000","last_change":"2024-04-26T04:59:45.906067+0000","last_active":"2024-04-26T04:59:45.906067+0000","last_peered":"2024-04-26T04:59:45.906067+0000","last_clean":"2024-04-26T04:59:45.906067+0000","last_became_active":"2024-04-26T04:59:45.902715+0000","last_became_peered":"2024-04-26T04:59:45.902715+0000","last_unstale":"2024-04-26T04:59:45.906067+0000","last_undegraded":"2024-04-26T04:59:45.906067+0000","last_fullsized":"2024-04-26T04:59:45.906067+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T14:50:51.953033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896147+0000","last_change":"2024-04-26T04:59:45.896147+0000","last_active":"2024-04-26T04:59:45.896147+0000","last_peered":"2024-04-26T04:59:45.896147+0000","last_clean":"2024-04-26T04:59:45.896147+0000","last_became_active":"2024-04-26T04:59:45.895858+0000","last_became_peered":"2024-04-26T04:59:45.895858+0000","last_unstale":"2024-04-26T04:59:45.896147+0000","last_undegraded":"2024-04-26T04:59:45.896147+0000","last_fullsized":"2024-04-26T04:59:45.896147+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:31:33.505661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887986+0000","last_change":"2024-04-26T04:54:53.658222+0000","last_active":"2024-04-26T04:59:45.887986+0000","last_peered":"2024-04-26T04:59:45.887986+0000","last_clean":"2024-04-26T04:59:45.887986+0000","last_became_active":"2024-04-26T04:54:53.656150+0000","last_became_peered":"2024-04-26T04:54:53.656150+0000","last_unstale":"2024-04-26T04:59:45.887986+0000","last_undegraded":"2024-04-26T04:59:45.887986+0000","last_fullsized":"2024-04-26T04:59:45.887986+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:22:25.670211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.898008+0000","last_change":"2024-04-26T04:59:45.898008+0000","last_active":"2024-04-26T04:59:45.898008+0000","last_peered":"2024-04-26T04:59:45.898008+0000","last_clean":"2024-04-26T04:59:45.898008+0000","last_became_active":"2024-04-26T04:59:45.897695+0000","last_became_peered":"2024-04-26T04:59:45.897695+0000","last_unstale":"2024-04-26T04:59:45.898008+0000","last_undegraded":"2024-04-26T04:59:45.898008+0000","last_fullsized":"2024-04-26T04:59:45.898008+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:30:40.519787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887575+0000","last_change":"2024-04-26T04:54:53.659780+0000","last_active":"2024-04-26T04:59:45.887575+0000","last_peered":"2024-04-26T04:59:45.887575+0000","last_clean":"2024-04-26T04:59:45.887575+0000","last_became_active":"2024-04-26T04:54:53.659228+0000","last_became_peered":"2024-04-26T04:54:53.659228+0000","last_unstale":"2024-04-26T04:59:45.887575+0000","last_undegraded":"2024-04-26T04:59:45.887575+0000","last_fullsized":"2024-04-26T04:59:45.887575+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:14:12.331200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889555+0000","last_change":"2024-04-26T04:54:53.655029+0000","last_active":"2024-04-26T04:59:44.889555+0000","last_peered":"2024-04-26T04:59:44.889555+0000","last_clean":"2024-04-26T04:59:44.889555+0000","last_became_active":"2024-04-26T04:54:53.654706+0000","last_became_peered":"2024-04-26T04:54:53.654706+0000","last_unstale":"2024-04-26T04:59:44.889555+0000","last_undegraded":"2024-04-26T04:59:44.889555+0000","last_fullsized":"2024-04-26T04:59:44.889555+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:19:18.843557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.898845+0000","last_change":"2024-04-26T04:51:40.698560+0000","last_active":"2024-04-26T04:59:44.898845+0000","last_peered":"2024-04-26T04:59:44.898845+0000","last_clean":"2024-04-26T04:59:44.898845+0000","last_became_active":"2024-04-26T04:51:40.698168+0000","last_became_peered":"2024-04-26T04:51:40.698168+0000","last_unstale":"2024-04-26T04:59:44.898845+0000","last_undegraded":"2024-04-26T04:59:44.898845+0000","last_fullsized":"2024-04-26T04:59:44.898845+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:13:49.122322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":120,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896060+0000","last_change":"2024-04-26T04:59:45.896060+0000","last_active":"2024-04-26T04:59:45.896060+0000","last_peered":"2024-04-26T04:59:45.896060+0000","last_clean":"2024-04-26T04:59:45.896060+0000","last_became_active":"2024-04-26T04:59:45.895737+0000","last_became_peered":"2024-04-26T04:59:45.895737+0000","last_unstale":"2024-04-26T04:59:45.896060+0000","last_undegraded":"2024-04-26T04:59:45.896060+0000","last_fullsized":"2024-04-26T04:59:45.896060+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:08:41.274761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":88,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.893896+0000","last_change":"2024-04-26T04:59:45.893896+0000","last_active":"2024-04-26T04:59:45.893896+0000","last_peered":"2024-04-26T04:59:45.893896+0000","last_clean":"2024-04-26T04:59:45.893896+0000","last_became_active":"2024-04-26T04:59:45.893525+0000","last_became_peered":"2024-04-26T04:59:45.893525+0000","last_unstale":"2024-04-26T04:59:45.893896+0000","last_undegraded":"2024-04-26T04:59:45.893896+0000","last_fullsized":"2024-04-26T04:59:45.893896+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:55:24.620601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888119+0000","last_change":"2024-04-26T04:59:09.197177+0000","last_active":"2024-04-26T04:59:45.888119+0000","last_peered":"2024-04-26T04:59:45.888119+0000","last_clean":"2024-04-26T04:59:45.888119+0000","last_became_active":"2024-04-26T04:59:09.196844+0000","last_became_peered":"2024-04-26T04:59:09.196844+0000","last_unstale":"2024-04-26T04:59:45.888119+0000","last_undegraded":"2024-04-26T04:59:45.888119+0000","last_fullsized":"2024-04-26T04:59:45.888119+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:09:21.125512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888619+0000","last_change":"2024-04-26T04:54:53.658857+0000","last_active":"2024-04-26T04:59:45.888619+0000","last_peered":"2024-04-26T04:59:45.888619+0000","last_clean":"2024-04-26T04:59:45.888619+0000","last_became_active":"2024-04-26T04:54:53.658476+0000","last_became_peered":"2024-04-26T04:54:53.658476+0000","last_unstale":"2024-04-26T04:59:45.888619+0000","last_undegraded":"2024-04-26T04:59:45.888619+0000","last_fullsized":"2024-04-26T04:59:45.888619+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:45:09.211999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.889480+0000","last_change":"2024-04-26T04:59:09.202553+0000","last_active":"2024-04-26T04:59:44.889480+0000","last_peered":"2024-04-26T04:59:44.889480+0000","last_clean":"2024-04-26T04:59:44.889480+0000","last_became_active":"2024-04-26T04:59:09.202205+0000","last_became_peered":"2024-04-26T04:59:09.202205+0000","last_unstale":"2024-04-26T04:59:44.889480+0000","last_undegraded":"2024-04-26T04:59:44.889480+0000","last_fullsized":"2024-04-26T04:59:44.889480+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:12:10.109522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"82'161","reported_seq":473,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.892164+0000","last_change":"2024-04-26T04:51:40.691091+0000","last_active":"2024-04-26T04:59:44.892164+0000","last_peered":"2024-04-26T04:59:44.892164+0000","last_clean":"2024-04-26T04:59:44.892164+0000","last_became_active":"2024-04-26T04:51:40.688992+0000","last_became_peered":"2024-04-26T04:51:40.688992+0000","last_unstale":"2024-04-26T04:59:44.892164+0000","last_undegraded":"2024-04-26T04:59:44.892164+0000","last_fullsized":"2024-04-26T04:59:44.892164+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:36:10.749042+0000","last_clean_scrub_stamp":"2024-04-26T04:36:10.749042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:52:04.890523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.903539+0000","last_change":"2024-04-26T04:59:45.903539+0000","last_active":"2024-04-26T04:59:45.903539+0000","last_peered":"2024-04-26T04:59:45.903539+0000","last_clean":"2024-04-26T04:59:45.903539+0000","last_became_active":"2024-04-26T04:59:45.902514+0000","last_became_peered":"2024-04-26T04:59:45.902514+0000","last_unstale":"2024-04-26T04:59:45.903539+0000","last_undegraded":"2024-04-26T04:59:45.903539+0000","last_fullsized":"2024-04-26T04:59:45.903539+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:11:49.800234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":74,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.895226+0000","last_change":"2024-04-26T04:39:56.923636+0000","last_active":"2024-04-26T04:59:44.895226+0000","last_peered":"2024-04-26T04:59:44.895226+0000","last_clean":"2024-04-26T04:59:44.895226+0000","last_became_active":"2024-04-26T04:39:56.923324+0000","last_became_peered":"2024-04-26T04:39:56.923324+0000","last_unstale":"2024-04-26T04:59:44.895226+0000","last_undegraded":"2024-04-26T04:59:44.895226+0000","last_fullsized":"2024-04-26T04:59:44.895226+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:05:54.903060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891469+0000","last_change":"2024-04-26T04:54:53.656591+0000","last_active":"2024-04-26T04:59:44.891469+0000","last_peered":"2024-04-26T04:59:44.891469+0000","last_clean":"2024-04-26T04:59:44.891469+0000","last_became_active":"2024-04-26T04:54:53.656008+0000","last_became_peered":"2024-04-26T04:54:53.656008+0000","last_unstale":"2024-04-26T04:59:44.891469+0000","last_undegraded":"2024-04-26T04:59:44.891469+0000","last_fullsized":"2024-04-26T04:59:44.891469+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:51:21.167848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906776+0000","last_change":"2024-04-26T04:59:45.905687+0000","last_active":"2024-04-26T04:59:45.906776+0000","last_peered":"2024-04-26T04:59:45.906776+0000","last_clean":"2024-04-26T04:59:45.906776+0000","last_became_active":"2024-04-26T04:59:45.905267+0000","last_became_peered":"2024-04-26T04:59:45.905267+0000","last_unstale":"2024-04-26T04:59:45.906776+0000","last_undegraded":"2024-04-26T04:59:45.906776+0000","last_fullsized":"2024-04-26T04:59:45.906776+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T08:49:14.504104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.890903+0000","last_change":"2024-04-26T04:59:09.197935+0000","last_active":"2024-04-26T04:59:45.890903+0000","last_peered":"2024-04-26T04:59:45.890903+0000","last_clean":"2024-04-26T04:59:45.890903+0000","last_became_active":"2024-04-26T04:59:09.197571+0000","last_became_peered":"2024-04-26T04:59:09.197571+0000","last_unstale":"2024-04-26T04:59:45.890903+0000","last_undegraded":"2024-04-26T04:59:45.890903+0000","last_fullsized":"2024-04-26T04:59:45.890903+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:01:39.998596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.891729+0000","last_change":"2024-04-26T04:51:40.781926+0000","last_active":"2024-04-26T04:59:44.891729+0000","last_peered":"2024-04-26T04:59:44.891729+0000","last_clean":"2024-04-26T04:59:44.891729+0000","last_became_active":"2024-04-26T04:51:40.781540+0000","last_became_peered":"2024-04-26T04:51:40.781540+0000","last_unstale":"2024-04-26T04:59:44.891729+0000","last_undegraded":"2024-04-26T04:59:44.891729+0000","last_fullsized":"2024-04-26T04:59:44.891729+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:43:44.834128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":111,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888119+0000","last_change":"2024-04-26T04:59:09.199954+0000","last_active":"2024-04-26T04:59:45.888119+0000","last_peered":"2024-04-26T04:59:45.888119+0000","last_clean":"2024-04-26T04:59:45.888119+0000","last_became_active":"2024-04-26T04:59:09.199312+0000","last_became_peered":"2024-04-26T04:59:09.199312+0000","last_unstale":"2024-04-26T04:59:45.888119+0000","last_undegraded":"2024-04-26T04:59:45.888119+0000","last_fullsized":"2024-04-26T04:59:45.888119+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:22:17.334848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":106,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887709+0000","last_change":"2024-04-26T04:51:40.708519+0000","last_active":"2024-04-26T04:59:45.887709+0000","last_peered":"2024-04-26T04:59:45.887709+0000","last_clean":"2024-04-26T04:59:45.887709+0000","last_became_active":"2024-04-26T04:51:40.702193+0000","last_became_peered":"2024-04-26T04:51:40.702193+0000","last_unstale":"2024-04-26T04:59:45.887709+0000","last_undegraded":"2024-04-26T04:59:45.887709+0000","last_fullsized":"2024-04-26T04:59:45.887709+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:53:20.759824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":67,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.906445+0000","last_change":"2024-04-26T04:59:45.906445+0000","last_active":"2024-04-26T04:59:45.906445+0000","last_peered":"2024-04-26T04:59:45.906445+0000","last_clean":"2024-04-26T04:59:45.906445+0000","last_became_active":"2024-04-26T04:59:45.906096+0000","last_became_peered":"2024-04-26T04:59:45.906096+0000","last_unstale":"2024-04-26T04:59:45.906445+0000","last_undegraded":"2024-04-26T04:59:45.906445+0000","last_fullsized":"2024-04-26T04:59:45.906445+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T16:18:15.331535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.887854+0000","last_change":"2024-04-26T04:54:53.660752+0000","last_active":"2024-04-26T04:59:45.887854+0000","last_peered":"2024-04-26T04:59:45.887854+0000","last_clean":"2024-04-26T04:59:45.887854+0000","last_became_active":"2024-04-26T04:54:53.659802+0000","last_became_peered":"2024-04-26T04:54:53.659802+0000","last_unstale":"2024-04-26T04:59:45.887854+0000","last_undegraded":"2024-04-26T04:59:45.887854+0000","last_fullsized":"2024-04-26T04:59:45.887854+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:34:12.694764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.902730+0000","last_change":"2024-04-26T04:59:45.902730+0000","last_active":"2024-04-26T04:59:45.902730+0000","last_peered":"2024-04-26T04:59:45.902730+0000","last_clean":"2024-04-26T04:59:45.902730+0000","last_became_active":"2024-04-26T04:59:45.901445+0000","last_became_peered":"2024-04-26T04:59:45.901445+0000","last_unstale":"2024-04-26T04:59:45.902730+0000","last_undegraded":"2024-04-26T04:59:45.902730+0000","last_fullsized":"2024-04-26T04:59:45.902730+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:53:56.765790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.894658+0000","last_change":"2024-04-26T04:59:09.198231+0000","last_active":"2024-04-26T04:59:44.894658+0000","last_peered":"2024-04-26T04:59:44.894658+0000","last_clean":"2024-04-26T04:59:44.894658+0000","last_became_active":"2024-04-26T04:59:09.197829+0000","last_became_peered":"2024-04-26T04:59:09.197829+0000","last_unstale":"2024-04-26T04:59:44.894658+0000","last_undegraded":"2024-04-26T04:59:44.894658+0000","last_fullsized":"2024-04-26T04:59:44.894658+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:38:53.035358+0000","last_clean_scrub_stamp":"2024-04-26T04:38:53.035358+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:13:00.464779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.890371+0000","last_change":"2024-04-26T04:54:53.655426+0000","last_active":"2024-04-26T04:59:45.890371+0000","last_peered":"2024-04-26T04:59:45.890371+0000","last_clean":"2024-04-26T04:59:45.890371+0000","last_became_active":"2024-04-26T04:54:53.653483+0000","last_became_peered":"2024-04-26T04:54:53.653483+0000","last_unstale":"2024-04-26T04:59:45.890371+0000","last_undegraded":"2024-04-26T04:59:45.890371+0000","last_fullsized":"2024-04-26T04:59:45.890371+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T12:13:21.331869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":74,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.899549+0000","last_change":"2024-04-26T04:39:56.315667+0000","last_active":"2024-04-26T04:59:44.899549+0000","last_peered":"2024-04-26T04:59:44.899549+0000","last_clean":"2024-04-26T04:59:44.899549+0000","last_became_active":"2024-04-26T04:39:56.315353+0000","last_became_peered":"2024-04-26T04:39:56.315353+0000","last_unstale":"2024-04-26T04:59:44.899549+0000","last_undegraded":"2024-04-26T04:59:44.899549+0000","last_fullsized":"2024-04-26T04:59:44.899549+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:45:00.659197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888041+0000","last_change":"2024-04-26T04:51:40.707690+0000","last_active":"2024-04-26T04:59:45.888041+0000","last_peered":"2024-04-26T04:59:45.888041+0000","last_clean":"2024-04-26T04:59:45.888041+0000","last_became_active":"2024-04-26T04:51:40.700784+0000","last_became_peered":"2024-04-26T04:51:40.700784+0000","last_unstale":"2024-04-26T04:59:45.888041+0000","last_undegraded":"2024-04-26T04:59:45.888041+0000","last_fullsized":"2024-04-26T04:59:45.888041+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:57:12.082663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":96,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.896140+0000","last_change":"2024-04-26T04:59:45.896140+0000","last_active":"2024-04-26T04:59:45.896140+0000","last_peered":"2024-04-26T04:59:45.896140+0000","last_clean":"2024-04-26T04:59:45.896140+0000","last_became_active":"2024-04-26T04:59:45.895738+0000","last_became_peered":"2024-04-26T04:59:45.895738+0000","last_unstale":"2024-04-26T04:59:45.896140+0000","last_undegraded":"2024-04-26T04:59:45.896140+0000","last_fullsized":"2024-04-26T04:59:45.896140+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T10:24:24.596759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":94,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.890835+0000","last_change":"2024-04-26T04:54:53.654261+0000","last_active":"2024-04-26T04:59:44.890835+0000","last_peered":"2024-04-26T04:59:44.890835+0000","last_clean":"2024-04-26T04:59:44.890835+0000","last_became_active":"2024-04-26T04:54:53.653828+0000","last_became_peered":"2024-04-26T04:54:53.653828+0000","last_unstale":"2024-04-26T04:59:44.890835+0000","last_undegraded":"2024-04-26T04:59:44.890835+0000","last_fullsized":"2024-04-26T04:59:44.890835+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T15:01:04.603924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900628+0000","last_change":"2024-04-26T04:59:45.900628+0000","last_active":"2024-04-26T04:59:45.900628+0000","last_peered":"2024-04-26T04:59:45.900628+0000","last_clean":"2024-04-26T04:59:45.900628+0000","last_became_active":"2024-04-26T04:59:45.900288+0000","last_became_peered":"2024-04-26T04:59:45.900288+0000","last_unstale":"2024-04-26T04:59:45.900628+0000","last_undegraded":"2024-04-26T04:59:45.900628+0000","last_fullsized":"2024-04-26T04:59:45.900628+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T13:25:45.956189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":77,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.892441+0000","last_change":"2024-04-26T04:39:56.299459+0000","last_active":"2024-04-26T04:59:44.892441+0000","last_peered":"2024-04-26T04:59:44.892441+0000","last_clean":"2024-04-26T04:59:44.892441+0000","last_became_active":"2024-04-26T04:39:56.299180+0000","last_became_peered":"2024-04-26T04:39:56.299180+0000","last_unstale":"2024-04-26T04:59:44.892441+0000","last_undegraded":"2024-04-26T04:59:44.892441+0000","last_fullsized":"2024-04-26T04:59:44.892441+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:51:01.512924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.900459+0000","last_change":"2024-04-26T04:59:45.900459+0000","last_active":"2024-04-26T04:59:45.900459+0000","last_peered":"2024-04-26T04:59:45.900459+0000","last_clean":"2024-04-26T04:59:45.900459+0000","last_became_active":"2024-04-26T04:59:45.900067+0000","last_became_peered":"2024-04-26T04:59:45.900067+0000","last_unstale":"2024-04-26T04:59:45.900459+0000","last_undegraded":"2024-04-26T04:59:45.900459+0000","last_fullsized":"2024-04-26T04:59:45.900459+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T11:28:11.952886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897624+0000","last_change":"2024-04-26T04:59:45.897624+0000","last_active":"2024-04-26T04:59:45.897624+0000","last_peered":"2024-04-26T04:59:45.897624+0000","last_clean":"2024-04-26T04:59:45.897624+0000","last_became_active":"2024-04-26T04:59:45.897151+0000","last_became_peered":"2024-04-26T04:59:45.897151+0000","last_unstale":"2024-04-26T04:59:45.897624+0000","last_undegraded":"2024-04-26T04:59:45.897624+0000","last_fullsized":"2024-04-26T04:59:45.897624+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T06:14:26.180049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.897199+0000","last_change":"2024-04-26T04:59:45.897199+0000","last_active":"2024-04-26T04:59:45.897199+0000","last_peered":"2024-04-26T04:59:45.897199+0000","last_clean":"2024-04-26T04:59:45.897199+0000","last_became_active":"2024-04-26T04:59:45.896884+0000","last_became_peered":"2024-04-26T04:59:45.896884+0000","last_unstale":"2024-04-26T04:59:45.897199+0000","last_undegraded":"2024-04-26T04:59:45.897199+0000","last_fullsized":"2024-04-26T04:59:45.897199+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T09:55:45.414733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":119,"reported_epoch":132,"state":"active+clean","last_fresh":"2024-04-26T04:59:45.888649+0000","last_change":"2024-04-26T04:59:09.199450+0000","last_active":"2024-04-26T04:59:45.888649+0000","last_peered":"2024-04-26T04:59:45.888649+0000","last_clean":"2024-04-26T04:59:45.888649+0000","last_became_active":"2024-04-26T04:59:09.199021+0000","last_became_peered":"2024-04-26T04:59:09.199021+0000","last_unstale":"2024-04-26T04:59:45.888649+0000","last_undegraded":"2024-04-26T04:59:45.888649+0000","last_fullsized":"2024-04-26T04:59:45.888649+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:55.248967+0000","last_clean_scrub_stamp":"2024-04-26T04:39:55.248967+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T07:56:38.218588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2024-04-26T04:59:44.893785+0000","last_change":"2024-04-26T04:54:53.662687+0000","last_active":"2024-04-26T04:59:44.893785+0000","last_peered":"2024-04-26T04:59:44.893785+0000","last_clean":"2024-04-26T04:59:44.893785+0000","last_became_active":"2024-04-26T04:54:53.662153+0000","last_became_peered":"2024-04-26T04:54:53.662153+0000","last_unstale":"2024-04-26T04:59:44.893785+0000","last_undegraded":"2024-04-26T04:59:44.893785+0000","last_fullsized":"2024-04-26T04:59:44.893785+0000","mapping_epoch":113,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":114,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-26T04:39:56.269145+0000","last_clean_scrub_stamp":"2024-04-26T04:39:56.269145+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-27T05:49:36.123381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13661,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201552,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307660,"kb_used_data":2112,"kb_used_omap":1,"kb_used_meta":305534,"kb_avail":93437492,"statfs":{"total":95995035648,"available":95679991808,"internally_reserved":0,"allocated":2162688,"data_stored":1429488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312867160},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463189,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311752,"kb_used_data":2108,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433400,"statfs":{"total":95995035648,"available":95675801600,"internally_reserved":0,"allocated":2158592,"data_stored":1429888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":113,"seq":485331304520,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":203576,"kb_used_data":1508,"kb_used_omap":5,"kb_used_meta":202042,"kb_avail":93541576,"statfs":{"total":95995035648,"available":95786573824,"internally_reserved":0,"allocated":1544192,"data_stored":834453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":206891287},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":91,"seq":390842024047,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":257988,"kb_used_data":2100,"kb_used_omap":5,"kb_used_meta":255866,"kb_avail":93487164,"statfs":{"total":95995035648,"available":95730855936,"internally_reserved":0,"allocated":2150400,"data_stored":1425694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6052,"internal_metadata":262006876},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879318,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":52412,"kb_used_data":1520,"kb_used_omap":6,"kb_used_meta":50873,"kb_avail":93692740,"statfs":{"total":95995035648,"available":95941365760,"internally_reserved":0,"allocated":1556480,"data_stored":835338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6732,"internal_metadata":52094388},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":131,"seq":562640715790,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77952,"kb_used_data":1516,"kb_used_omap":4,"kb_used_meta":76411,"kb_avail":93667200,"statfs":{"total":95995035648,"available":95915212800,"internally_reserved":0,"allocated":1552384,"data_stored":834816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4639,"internal_metadata":78245345},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986463,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599564,"kb_used_data":1532,"kb_used_omap":1,"kb_used_meta":598014,"kb_avail":93145588,"statfs":{"total":95995035648,"available":95381082112,"internally_reserved":0,"allocated":1568768,"data_stored":839331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612366680},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724825,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897208,"kb_used_data":1512,"kb_used_omap":1,"kb_used_meta":895678,"kb_avail":92847944,"statfs":{"total":95995035648,"available":95076294656,"internally_reserved":0,"allocated":1548288,"data_stored":834750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":917174616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4348,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2217,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2935,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-26T05:00:48.294 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-26T05:00:48.297 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-26T05:00:48.297 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-26T05:00:48.297 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:00:48.568 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:48.691 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:48 smithi038 ceph-mon[93172]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:48.691 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:48 smithi038 ceph-mon[96929]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:48 smithi055 ceph-mon[86906]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:49.679 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:49 smithi038 ceph-mon[93172]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:00:49.680 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:49 smithi038 ceph-mon[96929]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:00:49.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:49 smithi055 ceph-mon[86906]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-26T05:00:50.429 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:50.430 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:00:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:50 smithi038 ceph-mon[93172]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:50.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:50 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3744771593' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:00:50.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:50 smithi038 ceph-mon[96929]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:50.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:50 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3744771593' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:00:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:50 smithi055 ceph-mon[86906]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:50 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3744771593' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:00:52.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:52 smithi038 ceph-mon[93172]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:52 smithi038 ceph-mon[96929]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:52.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:52 smithi055 ceph-mon[86906]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:54.120 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:00:54.397 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:00:54.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:54 smithi038 ceph-mon[93172]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:54.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:54 smithi038 ceph-mon[96929]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:54.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:54 smithi055 ceph-mon[86906]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:56.379 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:00:56.379 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:00:56.670 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:56 smithi038 ceph-mon[93172]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:56.670 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:56 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3262127929' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:00:56.670 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:56 smithi038 ceph-mon[96929]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:56.671 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:56 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3262127929' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:00:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:56 smithi055 ceph-mon[86906]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:56 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3262127929' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:00:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:00:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:00:56] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:00:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:00:58 smithi055 ceph-mon[86906]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:58.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:00:58 smithi038 ceph-mon[93172]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:00:58.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:00:58 smithi038 ceph-mon[96929]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:00.006 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:00.284 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:00.673 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:00 smithi038 ceph-mon[93172]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:00.673 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:00 smithi038 ceph-mon[96929]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:00 smithi055 ceph-mon[86906]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:01.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:01 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:01.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:01 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:01.831 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:01 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:01:02.255 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:02.256 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:02.704 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:02 smithi038 ceph-mon[96929]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:02.704 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:02.704 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2882987749' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:02.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:02 smithi038 ceph-mon[93172]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:02.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:02.704 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2882987749' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:02.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:02 smithi055 ceph-mon[86906]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:02.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:02.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2882987749' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:03.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:03 smithi055 ceph-mon[86906]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:03.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:03 smithi038 ceph-mon[93172]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:03.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:03 smithi038 ceph-mon[96929]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.158 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:06.436 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[96929]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[93172]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:06 smithi055 ceph-mon[86906]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:06.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:06 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:01:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:01:06] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2024-04-26T05:01:07.577 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:07 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:07.577 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:07 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:07.577 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:07 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:07.577 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:07 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:07.577 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:07 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:07.578 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:07 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:07 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:01:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:07 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:01:07.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:07 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:01:08.383 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:08.383 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:08 smithi055 ceph-mon[86906]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:08 smithi055 ceph-mon[86906]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:08 smithi038 ceph-mon[93172]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:08 smithi038 ceph-mon[93172]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:08 smithi038 ceph-mon[96929]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:08.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:08 smithi038 ceph-mon[96929]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:09.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:09 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/273040774' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:09 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/273040774' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:09 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/273040774' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:10.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:10 smithi055 ceph-mon[86906]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:10.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:10 smithi038 ceph-mon[96929]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:10.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:10 smithi038 ceph-mon[93172]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:12.098 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:12.438 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:12.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:12 smithi038 ceph-mon[93172]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:12.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:12 smithi038 ceph-mon[96929]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:12 smithi055 ceph-mon[86906]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.360 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:14.360 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:14.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:14 smithi055 ceph-mon[86906]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.641 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:14 smithi038 ceph-mon[96929]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:14.641 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:14 smithi038 ceph-mon[93172]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:15.661 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:15 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2063212425' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:15 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2063212425' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:15 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2063212425' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:16.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:16 smithi038 ceph-mon[96929]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:16.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:16 smithi038 ceph-mon[93172]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:16.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:16 smithi055 ceph-mon[86906]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:01:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:01:16] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2024-04-26T05:01:17.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:17.989 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:18.265 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:18.720 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:18 smithi038 ceph-mon[93172]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.720 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:18 smithi038 ceph-mon[96929]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:18.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:18 smithi055 ceph-mon[86906]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:20.199 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:20.200 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:20 smithi038 ceph-mon[96929]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:20 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2295761026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:20 smithi038 ceph-mon[93172]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:20 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2295761026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:20 smithi055 ceph-mon[86906]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:20 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2295761026' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:22 smithi038 ceph-mon[93172]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:22 smithi038 ceph-mon[96929]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:22 smithi055 ceph-mon[86906]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:23.829 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:24.105 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:24.552 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:24 smithi038 ceph-mon[93172]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.552 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:24 smithi038 ceph-mon[96929]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:24 smithi055 ceph-mon[86906]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.031 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:26.031 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:26.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:26 smithi038 ceph-mon[93172]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.665 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1270529081' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:26.665 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:26 smithi038 ceph-mon[96929]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.665 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1270529081' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:26 smithi055 ceph-mon[86906]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1270529081' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:01:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:01:26] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2024-04-26T05:01:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:28 smithi038 ceph-mon[93172]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:28 smithi038 ceph-mon[96929]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:28 smithi055 ceph-mon[86906]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:29.693 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:29.975 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:30 smithi038 ceph-mon[96929]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:30 smithi038 ceph-mon[93172]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:30 smithi055 ceph-mon[86906]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:31.928 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:31.928 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:32.648 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:32 smithi038 ceph-mon[96929]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:32.648 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:32.648 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:32 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1981332738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:32.648 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:32 smithi038 ceph-mon[93172]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:32.648 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:32.648 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:32 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1981332738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:32 smithi055 ceph-mon[86906]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:32 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1981332738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:34.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:34 smithi038 ceph-mon[93172]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:34.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:34 smithi038 ceph-mon[96929]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:34 smithi055 ceph-mon[86906]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:35.681 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:35.958 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:36 smithi038 ceph-mon[93172]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:36 smithi038 ceph-mon[96929]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:36 smithi055 ceph-mon[86906]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:37.092 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:01:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:01:36] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:01:37.940 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:37.940 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:38.581 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:38 smithi038 ceph-mon[93172]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.582 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:38 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1558357491' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:38.582 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:38 smithi038 ceph-mon[96929]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.582 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:38 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1558357491' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:38 smithi055 ceph-mon[86906]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:38 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1558357491' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:40 smithi038 ceph-mon[93172]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:40 smithi038 ceph-mon[96929]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:40 smithi055 ceph-mon[86906]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:41.614 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:41.895 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:42 smithi038 ceph-mon[93172]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:42 smithi038 ceph-mon[96929]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:42 smithi055 ceph-mon[86906]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:44.082 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:44.082 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:44.839 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:44 smithi038 ceph-mon[93172]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:44.840 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3011665735' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:44.840 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:44 smithi038 ceph-mon[96929]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:44.841 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3011665735' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:44.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:44 smithi055 ceph-mon[86906]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:44.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3011665735' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:46.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:46 smithi055 ceph-mon[86906]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:46.931 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:46 smithi038 ceph-mon[93172]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:46.932 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:46 smithi038 ceph-mon[96929]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:01:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:01:46] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:01:47.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:47.874 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:47.901 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:47.901 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:01:48.152 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:48.601 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:48 smithi038 ceph-mon[93172]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.601 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:48 smithi038 ceph-mon[96929]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:48 smithi055 ceph-mon[86906]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:49.564 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:49 smithi038 ceph-mon[93172]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:49.565 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:49 smithi038 ceph-mon[96929]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:49.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:49 smithi055 ceph-mon[86906]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:50.218 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:50.218 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:50 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3436621608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:50.872 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:50 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3436621608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:50.872 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:50 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3436621608' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:51.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:51 smithi055 ceph-mon[86906]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:51.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:51 smithi038 ceph-mon[93172]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:51.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:51 smithi038 ceph-mon[96929]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:53.904 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:01:54.182 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:01:54.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:53 smithi038 ceph-mon[93172]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:54.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:53 smithi038 ceph-mon[96929]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:53 smithi055 ceph-mon[86906]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:56.091 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:01:56.091 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:01:56.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:55 smithi038 ceph-mon[93172]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:56.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:55 smithi038 ceph-mon[96929]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:55 smithi055 ceph-mon[86906]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:57.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:56 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/206653576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:57.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:56 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/206653576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:01:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:01:56] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:01:57.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:56 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/206653576' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:01:58.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:57 smithi055 ceph-mon[86906]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:57 smithi038 ceph-mon[93172]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:57 smithi038 ceph-mon[96929]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:01:59.807 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:00.089 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:00.212 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:01:59 smithi038 ceph-mon[93172]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:00.212 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:01:59 smithi038 ceph-mon[96929]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:01:59 smithi055 ceph-mon[86906]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:01.972 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:01.972 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:02 smithi038 ceph-mon[93172]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:02.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:02 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/548704913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:02.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:02 smithi038 ceph-mon[96929]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:02.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:02.225 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:02 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/548704913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:02 smithi055 ceph-mon[86906]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:02 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/548704913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:04 smithi055 ceph-mon[86906]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:04 smithi038 ceph-mon[93172]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:04 smithi038 ceph-mon[96929]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:05.623 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:05.902 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:06 smithi055 ceph-mon[86906]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:06.357 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:06 smithi038 ceph-mon[93172]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:06.358 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:06 smithi038 ceph-mon[96929]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:07.156 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:02:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:02:06] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:02:07.807 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:07.807 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:08.069 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:08 smithi038 ceph-mon[93172]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.069 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:08 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:08.069 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:08 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1162357698' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:08.070 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:08 smithi038 ceph-mon[96929]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.070 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:08 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:08.070 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:08 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1162357698' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:08 smithi055 ceph-mon[86906]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:08 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:02:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:08 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1162357698' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:09.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:09 smithi055 ceph-mon[86906]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:09.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:09 smithi038 ceph-mon[93172]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:09.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:09 smithi038 ceph-mon[96929]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:11.702 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:11.980 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:12.399 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:12.400 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:12.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.400 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:12.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:12.401 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:12.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:02:12.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:02:12.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:02:13.565 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:13 smithi038 ceph-mon[93172]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:13.565 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:13 smithi038 ceph-mon[93172]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:13.565 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:13 smithi038 ceph-mon[96929]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:13.565 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:13 smithi038 ceph-mon[96929]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:13.846 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:13.847 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:13 smithi055 ceph-mon[86906]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:13.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:13 smithi055 ceph-mon[86906]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:14.758 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:14 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/642183083' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:14.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:14 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/642183083' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:14.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:14 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/642183083' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:15.773 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:15 smithi055 ceph-mon[86906]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:15.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:15 smithi038 ceph-mon[96929]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:15.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:15 smithi038 ceph-mon[93172]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:02:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:02:16] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:02:17.530 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:17.808 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:17 smithi055 ceph-mon[86906]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:17.931 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:17 smithi038 ceph-mon[93172]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.931 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:17.931 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:17 smithi038 ceph-mon[96929]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:17.932 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:19.715 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:19.715 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:19.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:19 smithi055 ceph-mon[86906]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:19.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:19 smithi038 ceph-mon[93172]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:19.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:19 smithi038 ceph-mon[96929]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:20 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1461306130' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:20 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1461306130' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:20.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:20 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1461306130' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:21.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:21 smithi055 ceph-mon[86906]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:21 smithi038 ceph-mon[93172]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:21.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:21 smithi038 ceph-mon[96929]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:23.413 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:23.695 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:23.723 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:23 smithi038 ceph-mon[93172]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:23.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:23 smithi038 ceph-mon[96929]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:23.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:23 smithi055 ceph-mon[86906]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:25.569 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:25.569 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:25.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:25 smithi055 ceph-mon[86906]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:25.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:25 smithi038 ceph-mon[93172]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:25.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:25 smithi038 ceph-mon[96929]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:26.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:26 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4079201065' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:26.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:26 smithi038 ceph-mon[93172]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:26.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:26 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4079201065' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:26.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:26 smithi038 ceph-mon[96929]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:26 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4079201065' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:26 smithi055 ceph-mon[86906]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:02:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:02:26] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:02:29.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:29 smithi038 ceph-mon[93172]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:29 smithi038 ceph-mon[96929]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.483 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:29.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:29 smithi055 ceph-mon[86906]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:29.765 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:31.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:31 smithi038 ceph-mon[93172]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:31.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:31 smithi038 ceph-mon[96929]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:31.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:31 smithi055 ceph-mon[86906]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:31.693 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:31.694 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:32.383 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:32 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2590909200' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:32.383 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:32.383 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:32 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2590909200' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:32.384 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:32 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2590909200' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:32.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:33.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:33 smithi055 ceph-mon[86906]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:33 smithi038 ceph-mon[93172]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:33.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:33 smithi038 ceph-mon[96929]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:35.410 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:35.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:35 smithi055 ceph-mon[86906]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:35.686 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:35.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:35 smithi038 ceph-mon[93172]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:35.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:35 smithi038 ceph-mon[96929]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.097 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:02:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:02:36] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:02:37.352 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:37 smithi038 ceph-mon[96929]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.353 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:37 smithi038 ceph-mon[93172]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:37 smithi055 ceph-mon[86906]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:37.717 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:37.717 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:38.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:38 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/694052111' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:38.333 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:38 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/694052111' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:38 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/694052111' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:39.597 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:39 smithi038 ceph-mon[93172]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:39.597 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:39 smithi038 ceph-mon[96929]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:39.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:39 smithi055 ceph-mon[86906]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:41.361 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:41.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:41 smithi055 ceph-mon[86906]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:41.641 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:41.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:41 smithi038 ceph-mon[93172]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:41.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:41 smithi038 ceph-mon[96929]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:43.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:43 smithi038 ceph-mon[93172]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:43.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:43 smithi038 ceph-mon[96929]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:43.542 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:43.543 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:43.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:43 smithi055 ceph-mon[86906]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:44 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3386097011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:44.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:44 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3386097011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:44.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:44 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3386097011' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:45.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:45 smithi055 ceph-mon[86906]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:45.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:45 smithi038 ceph-mon[93172]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:45.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:45 smithi038 ceph-mon[96929]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:02:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:02:46] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:02:47.283 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:47.561 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:47 smithi055 ceph-mon[86906]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:47.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:47 smithi038 ceph-mon[93172]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:47.685 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:47 smithi038 ceph-mon[96929]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:47.685 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:02:49.328 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:49 smithi038 ceph-mon[96929]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:49.328 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:49 smithi038 ceph-mon[93172]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:49.598 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:49.599 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:49.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:49 smithi055 ceph-mon[86906]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:50 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1899823875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:50.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:50 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1899823875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:50 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1899823875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:51.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:51 smithi055 ceph-mon[86906]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:51.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:51 smithi038 ceph-mon[93172]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:51.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:51 smithi038 ceph-mon[96929]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:53.227 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:53.507 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:53 smithi055 ceph-mon[86906]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:53.624 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:53 smithi038 ceph-mon[96929]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:53.624 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:53 smithi038 ceph-mon[93172]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:55.499 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:55 smithi038 ceph-mon[93172]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:55.499 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:55 smithi038 ceph-mon[96929]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:55.516 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:02:55.517 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:02:55.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:55 smithi055 ceph-mon[86906]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:56.469 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:56 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/953965705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:56.469 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:56 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/953965705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:56.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:56 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/953965705' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:02:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:02:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:02:56] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:02:57.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:57 smithi055 ceph-mon[86906]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:57.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:57 smithi038 ceph-mon[93172]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:57.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:57 smithi038 ceph-mon[96929]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:59.125 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:02:59.406 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:02:59.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:02:59 smithi038 ceph-mon[93172]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:59.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:02:59 smithi038 ceph-mon[96929]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:02:59.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:02:59 smithi055 ceph-mon[86906]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:01.257 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:01.258 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:01.684 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:01 smithi038 ceph-mon[93172]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:01.684 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:01 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1098152166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:01.684 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:01 smithi038 ceph-mon[96929]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:01.684 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:01 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1098152166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:01 smithi055 ceph-mon[86906]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:01 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1098152166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:02.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:02.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:02.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:03.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:03 smithi038 ceph-mon[96929]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:03.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:03 smithi038 ceph-mon[93172]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:03.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:03 smithi055 ceph-mon[86906]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:04.931 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:05.208 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:05.641 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:05 smithi038 ceph-mon[96929]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:05.642 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:05 smithi038 ceph-mon[93172]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:05.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:05 smithi055 ceph-mon[86906]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:07.074 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:07.074 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:07.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:03:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:03:06] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:03:07.722 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:07 smithi038 ceph-mon[93172]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:07.722 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3113165335' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:07.723 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:07 smithi038 ceph-mon[96929]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:07.723 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3113165335' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:07.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:07 smithi055 ceph-mon[86906]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:07.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3113165335' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:09.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:09 smithi038 ceph-mon[93172]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:09.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:09 smithi038 ceph-mon[96929]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:09.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:09 smithi055 ceph-mon[86906]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:10.755 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:11.033 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:11.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:11 smithi038 ceph-mon[93172]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:11.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:11 smithi038 ceph-mon[96929]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:11.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:11 smithi055 ceph-mon[86906]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:12.468 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:12 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:12.469 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:12 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:12.845 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:12.845 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:12.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:12 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:03:13.522 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:13 smithi038 ceph-mon[96929]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:13.522 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:13 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2897791571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:13.523 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:13 smithi038 ceph-mon[93172]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:13.523 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:13 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2897791571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:13.832 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:13 smithi055 ceph-mon[86906]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:13.832 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:13 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2897791571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:15.707 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:15 smithi055 ceph-mon[86906]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:15.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:15 smithi038 ceph-mon[93172]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:15.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:15 smithi038 ceph-mon[96929]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:16.551 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:16.843 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:17.208 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:03:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:03:16] "GET /metrics HTTP/1.1" 200 33871 "" "Prometheus/2.43.0" 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[93172]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:17.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[96929]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.475 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:17 smithi055 ceph-mon[86906]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:17.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:18.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:18.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:18.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[93172]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.333 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[93172]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.334 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:18.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:18.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:18.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[96929]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[96929]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.334 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:03:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:03:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:18 smithi055 ceph-mon[86906]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:18 smithi055 ceph-mon[86906]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:18.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:03:19.025 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:19.026 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:19.322 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:19 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3772260498' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:19.323 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:19 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3772260498' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:19 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3772260498' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:20 smithi055 ceph-mon[86906]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:20 smithi038 ceph-mon[96929]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:20.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:20 smithi038 ceph-mon[93172]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.635 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:22.663 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:22 smithi038 ceph-mon[96929]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.664 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:22 smithi038 ceph-mon[93172]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:22 smithi055 ceph-mon[86906]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:22.913 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:24.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:24 smithi038 ceph-mon[96929]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:24.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:24 smithi038 ceph-mon[93172]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:24.761 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:24.761 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:24 smithi055 ceph-mon[86906]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:25.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:25 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/470985879' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:25.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:25 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/470985879' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:25.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:25 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/470985879' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:26.699 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:26 smithi038 ceph-mon[96929]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.699 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:26 smithi038 ceph-mon[93172]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:26 smithi055 ceph-mon[86906]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:26.974 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:03:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:03:26] "GET /metrics HTTP/1.1" 200 33871 "" "Prometheus/2.43.0" 2024-04-26T05:03:28.419 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:28.698 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:28 smithi038 ceph-mon[93172]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:28 smithi038 ceph-mon[96929]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:28 smithi055 ceph-mon[86906]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.593 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:30.593 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:30 smithi038 ceph-mon[93172]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:30 smithi038 ceph-mon[96929]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:30 smithi055 ceph-mon[86906]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:31.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:31 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3453247916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:31.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:31 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3453247916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:31.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:31 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3453247916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:32 smithi038 ceph-mon[93172]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:32 smithi038 ceph-mon[96929]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:32 smithi055 ceph-mon[86906]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:34.298 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:34.577 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:34.701 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:34 smithi038 ceph-mon[93172]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.701 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:34 smithi038 ceph-mon[96929]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:34 smithi055 ceph-mon[86906]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.409 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:36.410 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:36 smithi038 ceph-mon[93172]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/376027547' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:36 smithi038 ceph-mon[96929]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/376027547' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:36 smithi055 ceph-mon[86906]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/376027547' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:37.019 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:03:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:03:36] "GET /metrics HTTP/1.1" 200 33869 "" "Prometheus/2.43.0" 2024-04-26T05:03:38.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:38 smithi038 ceph-mon[93172]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:38.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:38 smithi038 ceph-mon[96929]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:38 smithi055 ceph-mon[86906]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.047 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:40.327 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:40.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:40 smithi038 ceph-mon[93172]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:40 smithi038 ceph-mon[96929]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:40 smithi055 ceph-mon[86906]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.294 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:42.295 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:42.508 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:42 smithi038 ceph-mon[96929]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.508 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:42 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4292965108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:42.508 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:42 smithi038 ceph-mon[93172]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.508 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:42 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4292965108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:42 smithi055 ceph-mon[86906]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:42 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4292965108' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:44.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:44 smithi038 ceph-mon[96929]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:44.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:44 smithi038 ceph-mon[93172]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:44.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:44 smithi055 ceph-mon[86906]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:46.028 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:46.310 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:46.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:46 smithi038 ceph-mon[93172]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:46.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:46 smithi038 ceph-mon[96929]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:46.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:46 smithi055 ceph-mon[86906]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:47.117 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:03:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:03:46] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2024-04-26T05:03:47.839 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:47.839 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:47.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:03:48.312 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:48.312 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:48.722 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:48 smithi038 ceph-mon[93172]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:48.722 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:48 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/909644097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:48.722 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:48 smithi038 ceph-mon[96929]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:48.722 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:48 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/909644097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:48 smithi055 ceph-mon[86906]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:48 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/909644097' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:50.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:50 smithi055 ceph-mon[86906]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:50.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:50 smithi038 ceph-mon[93172]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:50.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:50 smithi038 ceph-mon[96929]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:52.015 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:52.294 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:52.721 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:52 smithi038 ceph-mon[93172]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:52.721 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:52 smithi038 ceph-mon[96929]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:52.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:52 smithi055 ceph-mon[86906]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:54.361 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:03:54.361 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:03:54.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:54 smithi038 ceph-mon[93172]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:54.717 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:54 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3790627441' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:54.717 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:54 smithi038 ceph-mon[96929]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:54.717 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:54 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3790627441' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:54.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:54 smithi055 ceph-mon[86906]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:54.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:54 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3790627441' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:03:56.767 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:56 smithi038 ceph-mon[93172]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.767 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:56 smithi038 ceph-mon[96929]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:56.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:56 smithi055 ceph-mon[86906]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:03:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:03:56] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2024-04-26T05:03:58.065 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:03:58.339 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:03:58.709 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:03:58 smithi038 ceph-mon[93172]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.709 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:03:58 smithi038 ceph-mon[96929]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:03:58.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:03:58 smithi055 ceph-mon[86906]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.294 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:00.294 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:00.658 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:00 smithi038 ceph-mon[96929]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.658 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:00 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/166022302' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:00.659 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:00 smithi038 ceph-mon[93172]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.659 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:00 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/166022302' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:00 smithi055 ceph-mon[86906]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:00.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:00 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/166022302' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:01.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:01 smithi055 ceph-mon[86906]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:01.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:01 smithi038 ceph-mon[96929]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:01.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:01 smithi038 ceph-mon[93172]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:02.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:02.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:02.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:03.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:03 smithi055 ceph-mon[86906]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:03.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:03 smithi038 ceph-mon[93172]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:03.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:03 smithi038 ceph-mon[96929]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:03.998 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:04.276 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:06.165 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:06.165 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:06 smithi038 ceph-mon[93172]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:06.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:06 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2342758994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:06 smithi038 ceph-mon[96929]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:06.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:06 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2342758994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:06.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:06 smithi055 ceph-mon[86906]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:06.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:06 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2342758994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:07.220 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:04:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:04:06] "GET /metrics HTTP/1.1" 200 33873 "" "Prometheus/2.43.0" 2024-04-26T05:04:08.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:08 smithi038 ceph-mon[93172]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:08.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:08 smithi038 ceph-mon[96929]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:08.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:08 smithi055 ceph-mon[86906]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:09.868 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:10.147 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:10.592 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:10 smithi038 ceph-mon[93172]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:10.593 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:10 smithi038 ceph-mon[96929]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:10.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:10 smithi055 ceph-mon[86906]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.108 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:12.108 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:12 smithi038 ceph-mon[93172]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:12 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2987050508' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:12 smithi038 ceph-mon[96929]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:12 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2987050508' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:12.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:12 smithi055 ceph-mon[86906]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:12.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:12 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2987050508' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:14.658 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:14 smithi055 ceph-mon[86906]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:14.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:14 smithi038 ceph-mon[93172]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:14.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:14 smithi038 ceph-mon[96929]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:15.813 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:16.090 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:16.464 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:16 smithi038 ceph-mon[96929]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:16.464 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:16 smithi038 ceph-mon[93172]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:16.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:16 smithi055 ceph-mon[86906]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:17.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:04:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:04:16] "GET /metrics HTTP/1.1" 200 33871 "" "Prometheus/2.43.0" 2024-04-26T05:04:17.527 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:17.527 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:17.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:18.022 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:18.022 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:18.602 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:18 smithi055 ceph-mon[86906]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.603 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:18 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:18.603 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:18 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4281826281' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:18.655 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:18 smithi038 ceph-mon[93172]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.655 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:18 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:18.655 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:18 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4281826281' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:18.655 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:18 smithi038 ceph-mon[96929]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:18.655 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:18 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:04:18.656 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:18 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4281826281' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:20.699 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:20 smithi038 ceph-mon[93172]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:20.699 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:20 smithi038 ceph-mon[96929]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:20.811 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:20 smithi055 ceph-mon[86906]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:21.682 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:22.058 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[93172]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[96929]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:22.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:22 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:22 smithi055 ceph-mon[86906]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:22.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:22 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:23.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:23.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:23.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:23.725 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:23.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:23 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:04:23.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:23 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:04:23.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:23 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:04:23.981 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:23.981 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:24.674 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:24 smithi038 ceph-mon[93172]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.674 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:24 smithi038 ceph-mon[93172]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.674 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:24 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4284323112' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:24.675 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:24 smithi038 ceph-mon[96929]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.675 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:24 smithi038 ceph-mon[96929]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.675 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:24 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4284323112' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:24 smithi055 ceph-mon[86906]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:24 smithi055 ceph-mon[86906]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:24.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:24 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4284323112' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:26.777 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:26 smithi038 ceph-mon[93172]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:26.777 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:26 smithi038 ceph-mon[96929]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:26.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:26 smithi055 ceph-mon[86906]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:04:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:04:26] "GET /metrics HTTP/1.1" 200 33871 "" "Prometheus/2.43.0" 2024-04-26T05:04:27.705 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:27.984 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:28.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:28 smithi038 ceph-mon[93172]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:28.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:28 smithi038 ceph-mon[96929]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:28.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:28 smithi055 ceph-mon[86906]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.038 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:30.038 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:30 smithi038 ceph-mon[93172]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:30 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/84958322' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:30 smithi038 ceph-mon[96929]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:30 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/84958322' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:30 smithi055 ceph-mon[86906]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:30.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:30 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/84958322' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:32 smithi055 ceph-mon[86906]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:32.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:32 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:32.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:32 smithi038 ceph-mon[93172]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:32.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:32 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:32.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:32 smithi038 ceph-mon[96929]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:32.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:32 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:33.842 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:34.122 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:34.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:34 smithi055 ceph-mon[86906]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:34.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:34 smithi038 ceph-mon[96929]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:34.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:34 smithi038 ceph-mon[93172]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:36.145 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:36.145 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:36.808 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:36 smithi038 ceph-mon[93172]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:36.808 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:36 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/57978085' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:36.808 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:36 smithi038 ceph-mon[96929]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:36.808 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:36 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/57978085' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:36 smithi055 ceph-mon[86906]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:36.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:36 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/57978085' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:37.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:04:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:04:36] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:04:38.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:38 smithi055 ceph-mon[86906]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:38.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:38 smithi038 ceph-mon[93172]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:38.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:38 smithi038 ceph-mon[96929]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:39.836 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:40.115 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:40.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:40 smithi055 ceph-mon[86906]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:40.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:40 smithi038 ceph-mon[96929]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:40.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:40 smithi038 ceph-mon[93172]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:41.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:41 smithi055 ceph-mon[86906]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:41.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:41 smithi038 ceph-mon[93172]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:41.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:41 smithi038 ceph-mon[96929]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:42.066 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:42.067 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:42.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:42 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1296628471' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:42.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:42 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1296628471' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:42.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:42 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1296628471' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:43.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:43 smithi038 ceph-mon[93172]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:43.974 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:43 smithi038 ceph-mon[96929]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:44.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:43 smithi055 ceph-mon[86906]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:45.824 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:46.101 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:46.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:45 smithi038 ceph-mon[93172]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:46.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:45 smithi038 ceph-mon[96929]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:46.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:45 smithi055 ceph-mon[86906]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:47.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:46 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:47.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:46 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:47.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:04:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:04:46] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2024-04-26T05:04:47.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:46 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:04:48.189 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:48.189 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:48.189 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:47 smithi038 ceph-mon[93172]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:48.190 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:47 smithi038 ceph-mon[96929]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:48.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:47 smithi055 ceph-mon[86906]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:49.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:48 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1431128340' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:49.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:48 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1431128340' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:49.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:48 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1431128340' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:50.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:49 smithi055 ceph-mon[86906]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:50.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:49 smithi038 ceph-mon[93172]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:50.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:49 smithi038 ceph-mon[96929]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:51.909 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:52.193 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:04:52.223 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:52 smithi038 ceph-mon[96929]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:52.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:52 smithi038 ceph-mon[93172]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:52.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:52 smithi055 ceph-mon[86906]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:54.223 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:54 smithi038 ceph-mon[93172]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:54.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:54 smithi038 ceph-mon[96929]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:54.224 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:04:54.224 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:04:54.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:54 smithi055 ceph-mon[86906]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:55.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:55 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3146251794' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:55.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:55 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3146251794' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:55.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:55 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3146251794' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:04:56.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:56 smithi055 ceph-mon[86906]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:56.375 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:56 smithi038 ceph-mon[93172]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:56.375 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:56 smithi038 ceph-mon[96929]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:57.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:04:56 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:04:56] "GET /metrics HTTP/1.1" 200 33872 "" "Prometheus/2.43.0" 2024-04-26T05:04:58.200 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:04:58.357 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:04:58 smithi055 ceph-mon[86906]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:58.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:04:58 smithi038 ceph-mon[93172]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:58.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:04:58 smithi038 ceph-mon[96929]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:04:58.479 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:00.298 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:00 smithi038 ceph-mon[93172]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:00.298 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:00 smithi038 ceph-mon[96929]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:00.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:00 smithi055 ceph-mon[86906]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:00.562 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:00.562 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:01.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:01 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/770956077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:01.194 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:01 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/770956077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:01.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:01 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/770956077' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:02 smithi055 ceph-mon[86906]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:02.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:02 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:02 smithi038 ceph-mon[96929]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:02.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:02 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:02.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:02 smithi038 ceph-mon[93172]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:02.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:02 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:04.222 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:04.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:04 smithi055 ceph-mon[86906]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:04.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:04 smithi038 ceph-mon[93172]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:04.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:04 smithi038 ceph-mon[96929]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:04.501 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:06.157 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:06 smithi038 ceph-mon[93172]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:06.157 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:06 smithi038 ceph-mon[96929]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:06.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:06 smithi055 ceph-mon[86906]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:06.521 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:06.521 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:07.194 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:07 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/635459025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:07.194 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:06 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:05:06] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2024-04-26T05:05:07.194 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:07 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/635459025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:07.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:07 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/635459025' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:08.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:08 smithi055 ceph-mon[86906]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:08.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:08 smithi038 ceph-mon[93172]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:08.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:08 smithi038 ceph-mon[96929]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:10.222 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:10.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:10 smithi055 ceph-mon[86906]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:10.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:10 smithi038 ceph-mon[93172]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:10.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:10 smithi038 ceph-mon[96929]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:10.500 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:12.205 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:12 smithi038 ceph-mon[93172]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:12.205 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:12 smithi038 ceph-mon[96929]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:12.469 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:12.469 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:12.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:12 smithi055 ceph-mon[86906]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:13.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:13 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/2624598957' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:13.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:13 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/2624598957' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:13.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:13 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/2624598957' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:14.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:14 smithi038 ceph-mon[96929]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:14.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:14 smithi038 ceph-mon[93172]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:14.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:14 smithi055 ceph-mon[86906]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.224 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:16.408 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:16 smithi055 ceph-mon[86906]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:16 smithi038 ceph-mon[93172]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:16 smithi038 ceph-mon[96929]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:16.502 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:17.167 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:16 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:05:16] "GET /metrics HTTP/1.1" 200 33871 "" "Prometheus/2.43.0" 2024-04-26T05:05:17.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:17 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:17.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:17 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:17.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:17 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:18.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:18 smithi038 ceph-mon[96929]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:18.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:18 smithi038 ceph-mon[93172]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:18.457 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:18.457 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:18.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:18 smithi055 ceph-mon[86906]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:19.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:19 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3822250129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:19.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:19 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3822250129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:19.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:19 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3822250129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:20.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:20 smithi038 ceph-mon[93172]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:20.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:20 smithi038 ceph-mon[96929]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:20.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:20 smithi055 ceph-mon[86906]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:22.154 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:22.436 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:22.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:22 smithi038 ceph-mon[93172]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:22.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:22 smithi038 ceph-mon[96929]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:22.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:22 smithi055 ceph-mon[86906]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:23.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:23 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:23.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:23 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:23.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:23 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-26T05:05:24.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:24 smithi038 ceph-mon[93172]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:24.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:24 smithi038 ceph-mon[96929]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:24.556 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:24.556 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:24.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:24 smithi055 ceph-mon[86906]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:25.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:25 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3322659500' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:25.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:25 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3322659500' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:25.531 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:25 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3322659500' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:26.570 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:26 smithi055 ceph-mon[86906]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:26.586 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:26 smithi038 ceph-mon[93172]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:26.586 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:26 smithi038 ceph-mon[96929]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:27.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:05:26] "GET /metrics HTTP/1.1" 200 33871 "" "Prometheus/2.43.0" 2024-04-26T05:05:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:28 smithi038 ceph-mon[96929]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:28 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:28.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:28 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:28.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:28 smithi038 ceph-mon[93172]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:28.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:28 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:28.475 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:28 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:28.537 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:28 smithi055 ceph-mon[86906]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:28 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:28.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:28 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:28.812 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:29.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:29.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.857 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:29 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:29.974 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:29.975 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:29 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' 2024-04-26T05:05:30.817 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:30.818 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:31.201 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:30 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/4294120872' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:31.201 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:30 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/4294120872' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:31.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:30 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/4294120872' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:32.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:31 smithi038 ceph-mon[93172]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:32.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:31 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:32.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:31 smithi038 ceph-mon[96929]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:32.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:31 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:31 smithi055 ceph-mon[86906]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:32.356 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:31 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:34.544 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:34.573 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:34 smithi038 ceph-mon[93172]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:34.573 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:34 smithi038 ceph-mon[96929]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:34.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:34 smithi055 ceph-mon[86906]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:34.822 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:36.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:36 smithi055 ceph-mon[86906]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:36.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:36 smithi038 ceph-mon[93172]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:36.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:36 smithi038 ceph-mon[96929]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:36.818 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:36.818 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:37.130 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:36 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:05:36] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:05:37.474 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:37 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/3043232823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:37.474 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:37 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/3043232823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:37.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:37 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/3043232823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:38.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:38 smithi055 ceph-mon[86906]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:38.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:38 smithi038 ceph-mon[93172]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:38.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:38 smithi038 ceph-mon[96929]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:40.563 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:40.591 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:40 smithi038 ceph-mon[93172]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:40.592 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:40 smithi038 ceph-mon[96929]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:40.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:40 smithi055 ceph-mon[86906]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:40.841 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:42.593 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:42 smithi038 ceph-mon[93172]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:42.593 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:42 smithi038 ceph-mon[96929]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:42.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:42 smithi055 ceph-mon[86906]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:42.890 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:42.890 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:43.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:43 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/15538748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:43.640 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:43 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/15538748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:43.640 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:43 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/15538748' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:44.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:44 smithi055 ceph-mon[86906]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:44.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:44 smithi038 ceph-mon[93172]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:44.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:44 smithi038 ceph-mon[96929]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:46.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:46 smithi055 ceph-mon[86906]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:46.670 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid e22c1684-0385-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-26T05:05:46.697 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:46 smithi038 ceph-mon[93172]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:46.698 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:46 smithi038 ceph-mon[96929]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:46.948 INFO:teuthology.orchestra.run.smithi038.stderr:Inferring config /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/mon.a/config 2024-04-26T05:05:46.973 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:46 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: ::ffff:172.21.15.55 - - [26/Apr/2024:05:05:46] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2024-04-26T05:05:47.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:47 smithi055 ceph-mon[86906]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:47.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:47 smithi038 ceph-mon[93172]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:47.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:47 smithi038 ceph-mon[96929]: from='mgr.25231 172.21.15.38:0/2982080667' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-26T05:05:48.685 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:48 smithi038 ceph-mon[93172]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:48.685 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:48 smithi038 ceph-mon[96929]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:48.856 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:48 smithi055 ceph-mon[86906]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:49.049 INFO:teuthology.orchestra.run.smithi038.stdout: 2024-04-26T05:05:49.049 INFO:teuthology.orchestra.run.smithi038.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-26T05:05:49.724 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:49 smithi038 ceph-mon[93172]: from='client.? 172.21.15.38:0/1291879987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:49.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:49 smithi038 ceph-mon[96929]: from='client.? 172.21.15.38:0/1291879987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:49.823 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:05:50.061 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=81137905cac347419c6011b00041bde1 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_b22e2ebdeb24376882b7bda2a7329c8cccc2276a/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-26T05:05:50.064 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-26T05:05:50.073 INFO:tasks.cephadm:Teardown begin 2024-04-26T05:05:50.073 DEBUG:teuthology.orchestra.run.smithi038:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:05:50.077 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:49 smithi055 ceph-mon[86906]: from='client.? 172.21.15.38:0/1291879987' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-26T05:05:50.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:05:50.138 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-26T05:05:50.139 DEBUG:teuthology.orchestra.run.smithi038:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:05:50.160 DEBUG:teuthology.orchestra.run.smithi055:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-26T05:05:50.196 INFO:tasks.cephadm:Stopping all daemons... 2024-04-26T05:05:50.196 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-26T05:05:50.196 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a 2024-04-26T05:05:50.368 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:50 smithi038 systemd[1]: Stopping Ceph mon.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:05:50.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:50 smithi055 ceph-mon[86906]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:50.636 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-mon[96929]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:50.637 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-mon[93172]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-04-26T05:05:50.637 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[93168]: 2024-04-26T05:05:50.543+0000 7f2a0b358700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:50.637 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a[93168]: 2024-04-26T05:05:50.543+0000 7f2a0b358700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-26T05:05:50.637 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:05:05:50] ENGINE Bus STOPPING 2024-04-26T05:05:50.638 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:05:05:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-26T05:05:50.638 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:05:05:50] ENGINE Bus STOPPED 2024-04-26T05:05:50.638 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:05:05:50] ENGINE Bus STARTING 2024-04-26T05:05:50.963 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:05:05:50] ENGINE Serving on http://:::9283 2024-04-26T05:05:50.964 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: [26/Apr/2024:05:05:50] ENGINE Bus STARTED 2024-04-26T05:05:50.964 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:50 smithi038 podman[147678]: 2024-04-26 05:05:50.636295614 +0000 UTC m=+0.240435151 container died c46affca8e4e443fff694ab6eb4fb3ad0e534efdc57032a0f05252ae3ec149c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-26T05:05:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:51 smithi038 podman[147678]: 2024-04-26 05:05:51.161946537 +0000 UTC m=+0.766086113 container cleanup c46affca8e4e443fff694ab6eb4fb3ad0e534efdc57032a0f05252ae3ec149c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-04-26T05:05:51.224 INFO:journalctl@ceph.mon.a.smithi038.stdout:Apr 26 05:05:51 smithi038 bash[147678]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-a 2024-04-26T05:05:51.551 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.a.service' 2024-04-26T05:05:51.598 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:05:51.598 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-26T05:05:51.598 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-26T05:05:51.598 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.c 2024-04-26T05:05:51.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:51 smithi038 systemd[1]: Stopping Ceph mon.c for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:05:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c[96920]: 2024-04-26T05:05:51.890+0000 7f2689b44700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:51 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c[96920]: 2024-04-26T05:05:51.890+0000 7f2689b44700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:05:52.224 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:52 smithi038 podman[147811]: 2024-04-26 05:05:52.045354755 +0000 UTC m=+0.260511235 container died 7e20996682cea48c78cf2bad1aadbcae5df13439dcfaea319dc69c6b0e0c5a72 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5) 2024-04-26T05:05:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:52 smithi038 podman[147811]: 2024-04-26 05:05:52.417162867 +0000 UTC m=+0.632319405 container cleanup 7e20996682cea48c78cf2bad1aadbcae5df13439dcfaea319dc69c6b0e0c5a72 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-b22e2eb, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-26T05:05:52.724 INFO:journalctl@ceph.mon.c.smithi038.stdout:Apr 26 05:05:52 smithi038 bash[147811]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-c 2024-04-26T05:05:52.894 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.c.service' 2024-04-26T05:05:52.929 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:05:52.929 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-26T05:05:52.930 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-26T05:05:52.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.b 2024-04-26T05:05:53.241 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:52 smithi055 systemd[1]: Stopping Ceph mon.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:05:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b[86901]: 2024-04-26T05:05:53.239+0000 7f28aff29700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:53.606 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b[86901]: 2024-04-26T05:05:53.239+0000 7f28aff29700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-26T05:05:53.607 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:53 smithi055 podman[99070]: 2024-04-26 05:05:53.47567121 +0000 UTC m=+0.337211497 container died 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:05:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:54 smithi055 podman[99070]: 2024-04-26 05:05:54.007892228 +0000 UTC m=+0.869432515 container cleanup 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T05:05:54.106 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:54 smithi055 bash[99070]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b 2024-04-26T05:05:54.486 INFO:journalctl@ceph.mon.b.smithi055.stdout:Apr 26 05:05:54 smithi055 podman[99084]: 2024-04-26 05:05:54.234605766 +0000 UTC m=+0.750965259 container remove 7a992f3a10254467e57e9711730b05de14c72bb4fe72fa576c649255dbd3ec16 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb) 2024-04-26T05:05:54.490 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mon.b.service' 2024-04-26T05:05:54.543 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:05:54.544 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-26T05:05:54.544 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-26T05:05:54.544 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y 2024-04-26T05:05:54.848 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:54 smithi038 systemd[1]: Stopping Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:05:55.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T05:05:54.845+0000 7fa1ebdf5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:55.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y[83311]: 2024-04-26T05:05:54.845+0000 7fa1ebdf5700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-26T05:05:55.224 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:54 smithi038 podman[147932]: 2024-04-26 05:05:54.879405363 +0000 UTC m=+0.141259488 container died 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-26T05:05:55.676 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:55 smithi038 podman[147932]: 2024-04-26 05:05:55.40680565 +0000 UTC m=+0.668659783 container cleanup 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:05:55.676 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:55 smithi038 bash[147932]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y 2024-04-26T05:05:55.909 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-26T05:05:55.930 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:55 smithi038 podman[147946]: 2024-04-26 05:05:55.675425592 +0000 UTC m=+0.778946921 container remove 4c95c5f5ae0204a31949ebf65edf2994c3582210a1970dcdcb11dd99eed1ac63 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:05:55.931 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:55 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service: Deactivated successfully. 2024-04-26T05:05:55.931 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:55 smithi038 systemd[1]: Stopped Ceph mgr.y for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T05:05:55.931 INFO:journalctl@ceph.mgr.y.smithi038.stdout:Apr 26 05:05:55 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 15.341s CPU time. 2024-04-26T05:05:55.944 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:05:55.944 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-26T05:05:55.944 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-26T05:05:55.944 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x 2024-04-26T05:05:56.261 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 05:05:55 smithi055 systemd[1]: Stopping Ceph mgr.x for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:05:56.556 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 05:05:56 smithi055 podman[99194]: 2024-04-26 05:05:56.260672043 +0000 UTC m=+0.123044433 container died f26c41ca13263a58820f6d2481dddf1ea491be337c627a7b22a2a03abd99520f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-26T05:05:56.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 05:05:56 smithi055 podman[99194]: 2024-04-26 05:05:56.638592796 +0000 UTC m=+0.500965187 container cleanup f26c41ca13263a58820f6d2481dddf1ea491be337c627a7b22a2a03abd99520f (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True) 2024-04-26T05:05:56.856 INFO:journalctl@ceph.mgr.x.smithi055.stdout:Apr 26 05:05:56 smithi055 bash[99194]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-mgr-x 2024-04-26T05:05:57.061 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-26T05:05:57.095 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:05:57.095 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-26T05:05:57.095 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-26T05:05:57.096 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.0 2024-04-26T05:05:57.413 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:05:57 smithi038 systemd[1]: Stopping Ceph osd.0 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:05:57.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:05:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[105760]: 2024-04-26T05:05:57.410+0000 7f9aee24c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:05:57.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:05:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[105760]: 2024-04-26T05:05:57.410+0000 7f9aee24c700 -1 osd.0 132 *** Got signal Terminated *** 2024-04-26T05:05:57.724 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:05:57 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0[105760]: 2024-04-26T05:05:57.410+0000 7f9aee24c700 -1 osd.0 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:00.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:00.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:00.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:00.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:03.097 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:02 smithi038 podman[148053]: 2024-04-26 05:06:02.8273499 +0000 UTC m=+5.533847095 container died e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-26T05:06:03.673 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:03 smithi038 podman[148053]: 2024-04-26 05:06:03.396005349 +0000 UTC m=+6.102502612 container cleanup e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD) 2024-04-26T05:06:03.673 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:03 smithi038 bash[148053]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0 2024-04-26T05:06:03.974 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:03 smithi038 podman[148072]: 2024-04-26 05:06:03.672535776 +0000 UTC m=+0.841188643 container remove e5183e005cba46bec2d5742efd96affcc716d769384c47c8769cc01ffd268f51 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:04.235 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:03 smithi038 podman[148163]: 2024-04-26 05:06:03.887268148 +0000 UTC m=+0.018471260 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:04.235 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:04 smithi038 podman[148163]: 2024-04-26 05:06:04.006923694 +0000 UTC m=+0.138126829 container create 4f7fe18c6171fc976c3832bd0c3cd0bd5f52c7a8390134f40f18855d624aa032 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:04.235 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:04 smithi038 podman[148163]: 2024-04-26 05:06:04.234677898 +0000 UTC m=+0.365880995 container init 4f7fe18c6171fc976c3832bd0c3cd0bd5f52c7a8390134f40f18855d624aa032 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-26T05:06:04.536 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:04.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:04.536 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:04.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:04.537 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:04 smithi038 podman[148163]: 2024-04-26 05:06:04.238119182 +0000 UTC m=+0.369322289 container start 4f7fe18c6171fc976c3832bd0c3cd0bd5f52c7a8390134f40f18855d624aa032 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-26T05:06:04.537 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:04 smithi038 podman[148163]: 2024-04-26 05:06:04.351452662 +0000 UTC m=+0.482655777 container attach 4f7fe18c6171fc976c3832bd0c3cd0bd5f52c7a8390134f40f18855d624aa032 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:04.880 INFO:journalctl@ceph.osd.0.smithi038.stdout:Apr 26 05:06:04 smithi038 podman[148163]: 2024-04-26 05:06:04.536063467 +0000 UTC m=+0.667266564 container died 4f7fe18c6171fc976c3832bd0c3cd0bd5f52c7a8390134f40f18855d624aa032 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-b22e2eb, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T05:06:05.177 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.0.service' 2024-04-26T05:06:05.219 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:05.219 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-26T05:06:05.220 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-26T05:06:05.220 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.1 2024-04-26T05:06:05.474 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:05 smithi038 systemd[1]: Stopping Ceph osd.1 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:05.974 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[114210]: 2024-04-26T05:06:05.566+0000 7f2e9c320700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:05.974 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[114210]: 2024-04-26T05:06:05.566+0000 7f2e9c320700 -1 osd.1 132 *** Got signal Terminated *** 2024-04-26T05:06:05.974 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:05 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1[114210]: 2024-04-26T05:06:05.566+0000 7f2e9c320700 -1 osd.1 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:10.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:10.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:10.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:10.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:10.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:10 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:10.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:11.401 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:11 smithi038 podman[148270]: 2024-04-26 05:06:11.130879376 +0000 UTC m=+5.680927030 container died 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-04-26T05:06:11.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:11 smithi038 podman[148270]: 2024-04-26 05:06:11.584227318 +0000 UTC m=+6.134275033 container cleanup 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-26T05:06:11.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:11 smithi038 bash[148270]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1 2024-04-26T05:06:12.203 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:11 smithi038 podman[148288]: 2024-04-26 05:06:11.885871905 +0000 UTC m=+0.750851518 container remove 71e12d00a8a2aa01b37b1fc3f158970cc63f4db0981734396fbaa41c889a5c58 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-26T05:06:12.473 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:12 smithi038 podman[148379]: 2024-04-26 05:06:12.105844205 +0000 UTC m=+0.018660625 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:12.473 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:12 smithi038 podman[148379]: 2024-04-26 05:06:12.211842636 +0000 UTC m=+0.124659098 container create b2f984122db4afe449f6f3076114c75d3fd6bc82a3b92905452a81e14ccfb47d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424) 2024-04-26T05:06:12.473 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:12 smithi038 podman[148379]: 2024-04-26 05:06:12.397643717 +0000 UTC m=+0.310460111 container init b2f984122db4afe449f6f3076114c75d3fd6bc82a3b92905452a81e14ccfb47d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:12.473 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:12 smithi038 podman[148379]: 2024-04-26 05:06:12.400779807 +0000 UTC m=+0.313596203 container start b2f984122db4afe449f6f3076114c75d3fd6bc82a3b92905452a81e14ccfb47d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-26T05:06:12.473 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:12 smithi038 podman[148379]: 2024-04-26 05:06:12.472556656 +0000 UTC m=+0.385373059 container attach b2f984122db4afe449f6f3076114c75d3fd6bc82a3b92905452a81e14ccfb47d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-26T05:06:12.724 INFO:journalctl@ceph.osd.1.smithi038.stdout:Apr 26 05:06:12 smithi038 podman[148379]: 2024-04-26 05:06:12.707975511 +0000 UTC m=+0.620791907 container died b2f984122db4afe449f6f3076114c75d3fd6bc82a3b92905452a81e14ccfb47d (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb) 2024-04-26T05:06:13.390 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.1.service' 2024-04-26T05:06:13.431 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:13.431 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-26T05:06:13.431 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-26T05:06:13.432 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.2 2024-04-26T05:06:13.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:13 smithi038 systemd[1]: Stopping Ceph osd.2 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:14.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[123194]: 2024-04-26T05:06:13.771+0000 7fa112c0a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:14.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[123194]: 2024-04-26T05:06:13.771+0000 7fa112c0a700 -1 osd.2 132 *** Got signal Terminated *** 2024-04-26T05:06:14.224 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:13 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2[123194]: 2024-04-26T05:06:13.771+0000 7fa112c0a700 -1 osd.2 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:14.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:14.312Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:14.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:14.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:14.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:14 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:14.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:19.523 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:19 smithi038 podman[148483]: 2024-04-26 05:06:19.257635889 +0000 UTC m=+5.601988445 container died f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:19.857 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:19 smithi038 podman[148483]: 2024-04-26 05:06:19.688515434 +0000 UTC m=+6.032867984 container cleanup f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-04-26T05:06:19.857 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:19 smithi038 bash[148483]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2 2024-04-26T05:06:20.156 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:19 smithi038 podman[148501]: 2024-04-26 05:06:19.856610686 +0000 UTC m=+0.600865104 container remove f3f167abd6febfd3325ff8a7449fdefbd04fc1dee562c07f7f2b2d55575c2013 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:20.418 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:20 smithi038 podman[148591]: 2024-04-26 05:06:20.266136613 +0000 UTC m=+0.111369278 container create 3cf6f9c92f5ff38a432f9f415d0d6232011449197736ab18720cc369976d8c74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T05:06:20.418 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:20 smithi038 podman[148591]: 2024-04-26 05:06:20.173626741 +0000 UTC m=+0.018859434 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:20.419 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:20 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:20.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:20.419 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:20 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:20.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:20.419 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:20 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:20.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:20.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:20 smithi038 podman[148591]: 2024-04-26 05:06:20.52759765 +0000 UTC m=+0.372830320 container init 3cf6f9c92f5ff38a432f9f415d0d6232011449197736ab18720cc369976d8c74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-b22e2eb, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True) 2024-04-26T05:06:20.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:20 smithi038 podman[148591]: 2024-04-26 05:06:20.531034182 +0000 UTC m=+0.376266854 container start 3cf6f9c92f5ff38a432f9f415d0d6232011449197736ab18720cc369976d8c74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, RELEASE=squid-b22e2eb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-26T05:06:20.724 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:20 smithi038 podman[148591]: 2024-04-26 05:06:20.610434816 +0000 UTC m=+0.455667493 container attach 3cf6f9c92f5ff38a432f9f415d0d6232011449197736ab18720cc369976d8c74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-b22e2eb, ceph=True, CEPH_POINT_RELEASE=) 2024-04-26T05:06:21.130 INFO:journalctl@ceph.osd.2.smithi038.stdout:Apr 26 05:06:20 smithi038 podman[148591]: 2024-04-26 05:06:20.834653472 +0000 UTC m=+0.679886147 container died 3cf6f9c92f5ff38a432f9f415d0d6232011449197736ab18720cc369976d8c74 (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-b22e2eb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:21.506 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.2.service' 2024-04-26T05:06:21.551 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:21.552 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-26T05:06:21.552 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-26T05:06:21.552 DEBUG:teuthology.orchestra.run.smithi038:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.3 2024-04-26T05:06:21.784 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:21 smithi038 systemd[1]: Stopping Ceph osd.3 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:22.224 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:21 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T05:06:21.925+0000 7f6d102bc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:22.224 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:21 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T05:06:21.925+0000 7f6d102bc700 -1 osd.3 132 *** Got signal Terminated *** 2024-04-26T05:06:22.224 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:21 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T05:06:21.925+0000 7f6d102bc700 -1 osd.3 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:24.724 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:24 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T05:06:24.510+0000 7f6d088a8700 -1 osd.3 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.010190+0000 front 2024-04-26T05:06:01.010313+0000 (oldest deadline 2024-04-26T05:06:23.910329+0000) 2024-04-26T05:06:24.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:24 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:24.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:24.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:24 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:24.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:24.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:24 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:24.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:25.356 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:24 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:24.947+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:25.904 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:25 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:25.657+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:25.974 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:25 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T05:06:25.547+0000 7f6d088a8700 -1 osd.3 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.010190+0000 front 2024-04-26T05:06:01.010313+0000 (oldest deadline 2024-04-26T05:06:23.910329+0000) 2024-04-26T05:06:26.356 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:25 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:25.901+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:26.686 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:26.430+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:26.875 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:26 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3[128380]: 2024-04-26T05:06:26.590+0000 7f6d088a8700 -1 osd.3 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.010190+0000 front 2024-04-26T05:06:01.010313+0000 (oldest deadline 2024-04-26T05:06:23.910329+0000) 2024-04-26T05:06:27.106 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:26.683+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:27.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:26 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:26.922+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:27.724 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:27 smithi038 podman[148696]: 2024-04-26 05:06:27.467720619 +0000 UTC m=+5.674318316 container died db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:06:27.724 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:27.442+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:27.725 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:27.488+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:28.106 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:27.722+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:28.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:27 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:27.878+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:28.203 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:27 smithi038 podman[148696]: 2024-04-26 05:06:27.959778221 +0000 UTC m=+6.166375965 container cleanup db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-b22e2eb, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-26T05:06:28.203 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:27 smithi038 bash[148696]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3 2024-04-26T05:06:28.474 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:28 smithi038 podman[148724]: 2024-04-26 05:06:28.202711108 +0000 UTC m=+0.733964970 container remove db6a2a932dd3527861db155d4bb174624b13a6e2678a5b7b1ea34181dc85f23e (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-26T05:06:28.752 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:28.478+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:28.752 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:28.460+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:28.815 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:28 smithi038 podman[148817]: 2024-04-26 05:06:28.511842442 +0000 UTC m=+0.114751101 container create 6a02db366275fa57ee7cdd8f3c3fad2f7fe6cacf3c5d8d7916495e0306b1261c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T05:06:28.815 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:28 smithi038 podman[148817]: 2024-04-26 05:06:28.415625632 +0000 UTC m=+0.018534305 image pull 87e5280d5ae87e34552d9b448225db981ca5313b28020853b569ec2cb8ee53ae quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6 2024-04-26T05:06:28.815 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:28 smithi038 podman[148817]: 2024-04-26 05:06:28.731605482 +0000 UTC m=+0.334514142 container init 6a02db366275fa57ee7cdd8f3c3fad2f7fe6cacf3c5d8d7916495e0306b1261c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:28.816 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:28 smithi038 podman[148817]: 2024-04-26 05:06:28.734918707 +0000 UTC m=+0.337827369 container start 6a02db366275fa57ee7cdd8f3c3fad2f7fe6cacf3c5d8d7916495e0306b1261c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-b22e2eb, ceph=True, org.label-schema.build-date=20240424) 2024-04-26T05:06:28.816 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:28 smithi038 podman[148817]: 2024-04-26 05:06:28.81425769 +0000 UTC m=+0.417166365 container attach 6a02db366275fa57ee7cdd8f3c3fad2f7fe6cacf3c5d8d7916495e0306b1261c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-b22e2eb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-26T05:06:29.106 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:28.750+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:29.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:28 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:28.842+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:29.224 INFO:journalctl@ceph.osd.3.smithi038.stdout:Apr 26 05:06:29 smithi038 podman[148817]: 2024-04-26 05:06:29.038847483 +0000 UTC m=+0.641756147 container died 6a02db366275fa57ee7cdd8f3c3fad2f7fe6cacf3c5d8d7916495e0306b1261c (image=quay.ceph.io/ceph-ci/ceph@sha256:265587011b499c966b9dbaf00ce5756e85851e17df544df961d9e3203c8e39d6, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-b22e2eb, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T05:06:29.611 DEBUG:teuthology.orchestra.run.smithi038:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.3.service' 2024-04-26T05:06:29.655 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:29.655 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-26T05:06:29.655 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-26T05:06:29.655 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.4 2024-04-26T05:06:29.775 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:29 smithi055 systemd[1]: Stopping Ceph osd.4 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:29.776 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:29.511+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:29.776 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:29.504+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:30.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:29.824+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:30.107 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:29.773+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:30.107 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:29.965+0000 7fc5477f0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:30.107 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:29.965+0000 7fc5477f0700 -1 osd.4 132 *** Got signal Terminated *** 2024-04-26T05:06:30.107 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:29 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:29.965+0000 7fc5477f0700 -1 osd.4 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:30.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:30.157Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:30.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:30.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:30.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:30 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:30.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:30.782 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:30.543+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:30.783 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:30.492+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:31.106 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:30.781+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:31.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:30 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:30.804+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:31.826 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:31.541+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:31.826 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:31.575+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:31.827 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:31.575+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:32.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:31.854+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:32.107 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:31.825+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:32.107 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:31 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:31.825+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.953959+0000 front 2024-04-26T05:06:09.953941+0000 (oldest deadline 2024-04-26T05:06:31.653717+0000) 2024-04-26T05:06:32.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:06:32.057Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.38:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.38:8765: connect: connection refused" 2024-04-26T05:06:32.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:06:32.060Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.38:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.38:8765: connect: connection refused" 2024-04-26T05:06:32.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:06:32.060Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.38:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.38:8765: connect: connection refused" 2024-04-26T05:06:32.107 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:06:32.060Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.38:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.38:8765: connect: connection refused" 2024-04-26T05:06:32.108 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:06:32.060Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.38:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.38:8765: connect: connection refused" 2024-04-26T05:06:32.852 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:32.784+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:32.853 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:32.785+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.953959+0000 front 2024-04-26T05:06:09.953941+0000 (oldest deadline 2024-04-26T05:06:31.653717+0000) 2024-04-26T05:06:32.853 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:32.585+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:32.853 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:32.614+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:32.854 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:32.614+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:33.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:32 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:32.851+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:33.846 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:33.562+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:33.846 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:33.752+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:33.847 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:33.752+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.953959+0000 front 2024-04-26T05:06:09.953941+0000 (oldest deadline 2024-04-26T05:06:31.653717+0000) 2024-04-26T05:06:33.847 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:33.606+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:33.847 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:33.606+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:34.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:33 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:33.844+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:34.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:34.314Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:34.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:34.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:34.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:34 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:34.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:34.800 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:34.513+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:34.800 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:34.727+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:05:59.552099+0000 front 2024-04-26T05:05:59.551921+0000 (oldest deadline 2024-04-26T05:06:25.451856+0000) 2024-04-26T05:06:34.800 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4[35219]: 2024-04-26T05:06:34.727+0000 7fc540de8700 -1 osd.4 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.953959+0000 front 2024-04-26T05:06:09.953941+0000 (oldest deadline 2024-04-26T05:06:31.653717+0000) 2024-04-26T05:06:34.801 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:34.579+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:34.801 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:34.579+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:35.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:34 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:34.798+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:35.788 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:35 smithi055 podman[99333]: 2024-04-26 05:06:35.412403881 +0000 UTC m=+5.545819071 container died 40ece9b7ce4f0d31c9a6d3d4eb9440f968a3917b39a481c85f4758623797de2f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=quincy-c61472c) 2024-04-26T05:06:35.789 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:35.508+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:35.789 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:35.508+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:35.789 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:35.588+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:35.790 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:35.588+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:36.095 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:35 smithi055 podman[99333]: 2024-04-26 05:06:35.925768555 +0000 UTC m=+6.059183744 container cleanup 40ece9b7ce4f0d31c9a6d3d4eb9440f968a3917b39a481c85f4758623797de2f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5) 2024-04-26T05:06:36.095 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:35 smithi055 bash[99333]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4 2024-04-26T05:06:36.096 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:35.787+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:36.096 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:35 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:35.787+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:36.356 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:36 smithi055 podman[99351]: 2024-04-26 05:06:36.137471169 +0000 UTC m=+0.725369583 container remove 40ece9b7ce4f0d31c9a6d3d4eb9440f968a3917b39a481c85f4758623797de2f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:06:36.754 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:36.750+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:36.755 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:36.750+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:36.755 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:36.569+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:36.755 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:36.569+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:36.756 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:36 smithi055 podman[99448]: 2024-04-26 05:06:36.461710057 +0000 UTC m=+0.018546473 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:06:36.756 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:36 smithi055 podman[99448]: 2024-04-26 05:06:36.594640111 +0000 UTC m=+0.151476517 container create 4eeeb7b94ab3a89bbbc4e710f890c8bfbefbb75fdc832d8bf5d40d4e9bdf9020 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-26T05:06:36.756 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:36.500+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:36.756 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:36 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:36.500+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:37.106 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:36 smithi055 podman[99448]: 2024-04-26 05:06:36.989464727 +0000 UTC m=+0.546301134 container init 4eeeb7b94ab3a89bbbc4e710f890c8bfbefbb75fdc832d8bf5d40d4e9bdf9020 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-26T05:06:37.106 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:36 smithi055 podman[99448]: 2024-04-26 05:06:36.992800676 +0000 UTC m=+0.549637090 container start 4eeeb7b94ab3a89bbbc4e710f890c8bfbefbb75fdc832d8bf5d40d4e9bdf9020 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-26T05:06:37.508 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:37 smithi055 podman[99448]: 2024-04-26 05:06:37.135037004 +0000 UTC m=+0.691873427 container attach 4eeeb7b94ab3a89bbbc4e710f890c8bfbefbb75fdc832d8bf5d40d4e9bdf9020 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-26T05:06:37.508 INFO:journalctl@ceph.osd.4.smithi055.stdout:Apr 26 05:06:37 smithi055 podman[99448]: 2024-04-26 05:06:37.299557575 +0000 UTC m=+0.856393981 container died 4eeeb7b94ab3a89bbbc4e710f890c8bfbefbb75fdc832d8bf5d40d4e9bdf9020 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:06:37.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:37.506+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:37.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:37.506+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:37.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:37.722+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:37.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:37.722+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:37.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:37.551+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:37.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:37 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:37.551+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:37.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.4.service' 2024-04-26T05:06:37.971 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:37.971 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-26T05:06:37.971 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-26T05:06:37.971 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.5 2024-04-26T05:06:38.324 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:38 smithi055 systemd[1]: Stopping Ceph osd.5 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:38.324 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:38.322+0000 7fd506850700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:38.324 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:38.322+0000 7fd506850700 -1 osd.5 132 *** Got signal Terminated *** 2024-04-26T05:06:38.324 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:38.322+0000 7fd506850700 -1 osd.5 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:38.574 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:38.515+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:38.574 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:38.515+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:38.856 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:38.699+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:38.856 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:38.699+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:38.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:38.582+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:38.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:38 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:38.582+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:39.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:39.517+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:39.857 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:39.517+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:39.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:39.658+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:39.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:39.658+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:39.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:39.539+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:39.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:39 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:39.539+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:40.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:40.157Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:40.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:40.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:40.475 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:40 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:40.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:40.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:40.493+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:40.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:40.493+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:40.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:40.638+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:40.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:40.638+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:40.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:40.638+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:40.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:40.546+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:40.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:40 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:40.546+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:41.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:41.456+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:41.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:41.456+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:41.857 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:41.456+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.075352+0000 front 2024-04-26T05:06:15.075299+0000 (oldest deadline 2024-04-26T05:06:40.975059+0000) 2024-04-26T05:06:41.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:41.580+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:41.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:41.580+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:41.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:41.678+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:41.858 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:41.678+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:41.858 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:41 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:41.678+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:42.856 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:42.470+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:02.173829+0000 front 2024-04-26T05:06:02.173698+0000 (oldest deadline 2024-04-26T05:06:25.673591+0000) 2024-04-26T05:06:42.857 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:42.470+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.774554+0000 front 2024-04-26T05:06:09.774569+0000 (oldest deadline 2024-04-26T05:06:35.074596+0000) 2024-04-26T05:06:42.857 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5[41333]: 2024-04-26T05:06:42.470+0000 7fd4ffe48700 -1 osd.5 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.075352+0000 front 2024-04-26T05:06:15.075299+0000 (oldest deadline 2024-04-26T05:06:40.975059+0000) 2024-04-26T05:06:42.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:42.619+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:42.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:42.619+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:42.858 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:42.675+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:42.858 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:42.675+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:42.858 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:42 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:42.675+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:44.106 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:43 smithi055 podman[99564]: 2024-04-26 05:06:43.869223952 +0000 UTC m=+5.649164494 container died 50fe68fbb90591ec87249e482f961523b63307155f6f3d5d30cdba9e784e10e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-04-26T05:06:44.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:43.619+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:44.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:43.619+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:44.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:43.619+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:44.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:43.701+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:44.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:43.701+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:44.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:43 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:43.701+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:44.517 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:44 smithi055 podman[99564]: 2024-04-26 05:06:44.21453325 +0000 UTC m=+5.994473797 container cleanup 50fe68fbb90591ec87249e482f961523b63307155f6f3d5d30cdba9e784e10e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:44.517 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:44 smithi055 bash[99564]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5 2024-04-26T05:06:44.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:44 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:44.315Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:44.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:44 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:44.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:44.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:44 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:44.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:44.817 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:44 smithi055 podman[99582]: 2024-04-26 05:06:44.516171286 +0000 UTC m=+0.650304292 container remove 50fe68fbb90591ec87249e482f961523b63307155f6f3d5d30cdba9e784e10e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_CLEAN=True, RELEASE=quincy-c61472c, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-04-26T05:06:44.817 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:44.672+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:44.818 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:44.673+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:44.818 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:44.673+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:44.818 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:44.580+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:44.818 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:44.580+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:44.818 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:44 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:44.580+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:45.070 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:44 smithi055 podman[99671]: 2024-04-26 05:06:44.817015013 +0000 UTC m=+0.107999072 container create 6f356dbfc0db86db5d7eae6ddff0dddcd3bd4bdaf9b254054bcec2b70d135586 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c) 2024-04-26T05:06:45.070 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:44 smithi055 podman[99671]: 2024-04-26 05:06:44.72751917 +0000 UTC m=+0.018503245 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:06:45.071 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:45 smithi055 podman[99671]: 2024-04-26 05:06:45.070012786 +0000 UTC m=+0.360996843 container init 6f356dbfc0db86db5d7eae6ddff0dddcd3bd4bdaf9b254054bcec2b70d135586 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:06:45.356 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:45 smithi055 podman[99671]: 2024-04-26 05:06:45.073383368 +0000 UTC m=+0.364367425 container start 6f356dbfc0db86db5d7eae6ddff0dddcd3bd4bdaf9b254054bcec2b70d135586 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:45.356 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:45 smithi055 podman[99671]: 2024-04-26 05:06:45.144841971 +0000 UTC m=+0.435826028 container attach 6f356dbfc0db86db5d7eae6ddff0dddcd3bd4bdaf9b254054bcec2b70d135586 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-26T05:06:45.640 INFO:journalctl@ceph.osd.5.smithi055.stdout:Apr 26 05:06:45 smithi055 podman[99671]: 2024-04-26 05:06:45.361345245 +0000 UTC m=+0.652329302 container died 6f356dbfc0db86db5d7eae6ddff0dddcd3bd4bdaf9b254054bcec2b70d135586 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c) 2024-04-26T05:06:45.640 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:45.546+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:45.640 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:45.546+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:45.640 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:45.546+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:45.916 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:45.717+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:45.916 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:45.717+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:45.916 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:45 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:45.717+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:45.932 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.5.service' 2024-04-26T05:06:45.974 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:45.975 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-26T05:06:45.975 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-26T05:06:45.975 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.6 2024-04-26T05:06:46.320 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 systemd[1]: Stopping Ceph osd.6 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:46.320 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:46.318+0000 7f763675a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:46.320 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:46.318+0000 7f763675a700 -1 osd.6 132 *** Got signal Terminated *** 2024-04-26T05:06:46.320 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:46.318+0000 7f763675a700 -1 osd.6 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:46.856 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:46.713+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:46.856 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:46.713+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:46.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:46.713+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:46.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:46.586+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:46.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:46.586+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:46.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:46 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:46.586+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:48.034 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:47.674+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:48.034 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:47.674+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:48.034 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:47.674+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:48.035 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:47.605+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:48.035 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:47.605+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:48.035 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:47 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:47.605+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:49.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:48.639+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:49.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:48.639+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:49.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:48.639+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:49.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:48.618+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:49.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:48.618+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:49.108 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:48.618+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:49.108 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:48 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:48.618+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:50.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:49.636+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:50.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:49.636+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:50.107 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:49.636+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:50.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:49.608+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:50.108 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:49.608+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:50.108 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:49.608+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:50.108 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:49 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:49.608+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:50.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:50.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:50.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:50.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:50.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:50 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:50.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:50.856 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:50.598+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.593319+0000 front 2024-04-26T05:06:01.593515+0000 (oldest deadline 2024-04-26T05:06:24.493260+0000) 2024-04-26T05:06:50.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:50.598+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:09.794246+0000 front 2024-04-26T05:06:09.794189+0000 (oldest deadline 2024-04-26T05:06:35.694185+0000) 2024-04-26T05:06:50.857 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6[47365]: 2024-04-26T05:06:50.598+0000 7f7633d5a700 -1 osd.6 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:15.694989+0000 front 2024-04-26T05:06:15.695114+0000 (oldest deadline 2024-04-26T05:06:40.394612+0000) 2024-04-26T05:06:50.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:50.642+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:50.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:50.642+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:50.857 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:50.642+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:50.858 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:50 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:50.642+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:52.106 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:51 smithi055 podman[99777]: 2024-04-26 05:06:51.931610984 +0000 UTC m=+5.726150355 container died bc76cfe8fbcd4cb597e6b6edd8b0aa2600a8c84e3177d780cc872f2d229a0275 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, CEPH_POINT_RELEASE=) 2024-04-26T05:06:52.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:51 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:51.691+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:52.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:51 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:51.691+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:52.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:51 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:51.691+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:52.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:51 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:51.691+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:52.454 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:52 smithi055 podman[99777]: 2024-04-26 05:06:52.277039786 +0000 UTC m=+6.071579199 container cleanup bc76cfe8fbcd4cb597e6b6edd8b0aa2600a8c84e3177d780cc872f2d229a0275 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-04-26T05:06:52.454 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:52 smithi055 bash[99777]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6 2024-04-26T05:06:52.734 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:52 smithi055 podman[99797]: 2024-04-26 05:06:52.453305464 +0000 UTC m=+0.515882247 container remove bc76cfe8fbcd4cb597e6b6edd8b0aa2600a8c84e3177d780cc872f2d229a0275 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-26T05:06:53.057 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:52 smithi055 podman[99888]: 2024-04-26 05:06:52.807112875 +0000 UTC m=+0.018353375 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:06:53.058 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:52 smithi055 podman[99888]: 2024-04-26 05:06:52.937842714 +0000 UTC m=+0.149083206 container create 68b770365f1c60e8dc79fdb7440a288dcdb607c77b6747782fa6cd792d612e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=quincy-c61472c, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:06:53.058 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:52 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:52.732+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:53.058 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:52 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:52.732+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:53.058 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:52 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:52.732+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:53.058 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:52 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:52.732+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:53.356 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 podman[99888]: 2024-04-26 05:06:53.165804929 +0000 UTC m=+0.377045424 container init 68b770365f1c60e8dc79fdb7440a288dcdb607c77b6747782fa6cd792d612e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-26T05:06:53.357 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 podman[99888]: 2024-04-26 05:06:53.169255687 +0000 UTC m=+0.380496185 container start 68b770365f1c60e8dc79fdb7440a288dcdb607c77b6747782fa6cd792d612e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-c61472c, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:53.357 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 podman[99888]: 2024-04-26 05:06:53.248997463 +0000 UTC m=+0.460237963 container attach 68b770365f1c60e8dc79fdb7440a288dcdb607c77b6747782fa6cd792d612e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-c61472c) 2024-04-26T05:06:53.744 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 podman[99888]: 2024-04-26 05:06:53.455345514 +0000 UTC m=+0.666586013 container died 68b770365f1c60e8dc79fdb7440a288dcdb607c77b6747782fa6cd792d612e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-26T05:06:53.745 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:53.692+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:53.745 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:53.692+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:53.745 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:53.692+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:53.745 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:53 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:53.692+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:53.987 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.6.service' 2024-04-26T05:06:54.012 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 podman[99915]: 2024-04-26 05:06:53.961344281 +0000 UTC m=+0.500839958 container remove 68b770365f1c60e8dc79fdb7440a288dcdb607c77b6747782fa6cd792d612e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-c61472c, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.schema-version=1.0) 2024-04-26T05:06:54.012 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-26T05:06:54.013 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 systemd[1]: Stopped Ceph osd.6 for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T05:06:54.013 INFO:journalctl@ceph.osd.6.smithi055.stdout:Apr 26 05:06:53 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.6.service: Consumed 18min 21.687s CPU time. 2024-04-26T05:06:54.029 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:06:54.029 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-26T05:06:54.029 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-26T05:06:54.029 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.7 2024-04-26T05:06:54.356 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 systemd[1]: Stopping Ceph osd.7 for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:06:54.689 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.355+0000 7f09f5c4f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-26T05:06:54.690 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.355+0000 7f09f5c4f700 -1 osd.7 132 *** Got signal Terminated *** 2024-04-26T05:06:54.690 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.355+0000 7f09f5c4f700 -1 osd.7 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-26T05:06:54.690 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.688+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:54.690 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.688+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:54.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:54.315Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:54.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:54.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:06:54.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:06:54 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:06:54.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:06:55.106 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.688+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:55.106 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:54 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:54.688+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:56.008 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:55 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:55.704+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:56.008 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:55 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:55.704+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:56.008 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:55 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:55.704+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:56.008 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:55 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:55.704+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:57.024 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:56.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:57.025 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:56.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:57.025 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:56.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:57.025 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:56 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:56.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:58.106 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:57.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:58.106 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:57.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:58.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:57.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:58.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:57.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:58.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:57 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:57.671+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.55:6804 osd.4 since back 2024-04-26T05:06:32.924686+0000 front 2024-04-26T05:06:32.924594+0000 (oldest deadline 2024-04-26T05:06:57.624488+0000) 2024-04-26T05:06:59.106 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:58.629+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6806 osd.0 since back 2024-04-26T05:06:01.920955+0000 front 2024-04-26T05:06:01.920849+0000 (oldest deadline 2024-04-26T05:06:26.620726+0000) 2024-04-26T05:06:59.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:58.629+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6814 osd.1 since back 2024-04-26T05:06:06.621172+0000 front 2024-04-26T05:06:06.621279+0000 (oldest deadline 2024-04-26T05:06:31.321330+0000) 2024-04-26T05:06:59.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:58.629+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6822 osd.2 since back 2024-04-26T05:06:14.822586+0000 front 2024-04-26T05:06:14.822540+0000 (oldest deadline 2024-04-26T05:06:43.022993+0000) 2024-04-26T05:06:59.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:58.629+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.38:6830 osd.3 since back 2024-04-26T05:06:25.923740+0000 front 2024-04-26T05:06:25.923684+0000 (oldest deadline 2024-04-26T05:06:47.623785+0000) 2024-04-26T05:06:59.107 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:58 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7[53427]: 2024-04-26T05:06:58.629+0000 7f09f3a50700 -1 osd.7 132 heartbeat_check: no reply from 172.21.15.55:6804 osd.4 since back 2024-04-26T05:06:32.924686+0000 front 2024-04-26T05:06:32.924594+0000 (oldest deadline 2024-04-26T05:06:57.624488+0000) 2024-04-26T05:07:00.049 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:06:59 smithi055 podman[99993]: 2024-04-26 05:06:59.762851337 +0000 UTC m=+5.491558687 container died 6ca051d18a67bda032d52242ba7ef35ecf56f3b1337e5ec4f926d59bd9405570 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-04-26T05:07:00.356 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 podman[99993]: 2024-04-26 05:07:00.131353319 +0000 UTC m=+5.860060668 container cleanup 6ca051d18a67bda032d52242ba7ef35ecf56f3b1337e5ec4f926d59bd9405570 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7, io.buildah.version=1.33.5, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-26T05:07:00.356 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 bash[99993]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7 2024-04-26T05:07:00.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:00.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:07:00.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:00.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:07:00.474 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:00 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:00.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:07:00.726 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 podman[100024]: 2024-04-26 05:07:00.407822876 +0000 UTC m=+0.646195383 container remove 6ca051d18a67bda032d52242ba7ef35ecf56f3b1337e5ec4f926d59bd9405570 (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-26T05:07:01.011 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 podman[100114]: 2024-04-26 05:07:00.629213652 +0000 UTC m=+0.018503358 image pull 0de407c1b3afb7a571e0cc4b0c826a995ed358d49d5f013ca6bdfad025b55c43 quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17 2024-04-26T05:07:01.011 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 podman[100114]: 2024-04-26 05:07:00.73360699 +0000 UTC m=+0.122896683 container create e0469349ffec889476e04f315b5482d0ecc60e051f330f7b7b0ffb8d1022e57a (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-c61472c, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67) 2024-04-26T05:07:01.012 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 podman[100114]: 2024-04-26 05:07:00.927783592 +0000 UTC m=+0.317073286 container init e0469349ffec889476e04f315b5482d0ecc60e051f330f7b7b0ffb8d1022e57a (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, ceph=True) 2024-04-26T05:07:01.012 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:00 smithi055 podman[100114]: 2024-04-26 05:07:00.93089207 +0000 UTC m=+0.320181766 container start e0469349ffec889476e04f315b5482d0ecc60e051f330f7b7b0ffb8d1022e57a (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=quincy-c61472c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-04-26T05:07:01.012 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:01 smithi055 podman[100114]: 2024-04-26 05:07:01.011207785 +0000 UTC m=+0.400497484 container attach e0469349ffec889476e04f315b5482d0ecc60e051f330f7b7b0ffb8d1022e57a (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7-deactivate, ceph=True, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, RELEASE=quincy-c61472c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-26T05:07:01.356 INFO:journalctl@ceph.osd.7.smithi055.stdout:Apr 26 05:07:01 smithi055 podman[100114]: 2024-04-26 05:07:01.21932918 +0000 UTC m=+0.608618873 container died e0469349ffec889476e04f315b5482d0ecc60e051f330f7b7b0ffb8d1022e57a (image=quay.ceph.io/ceph-ci/ceph@sha256:27d43f80a23d3545096a7bb490598a6f37852da2940e69e5a8f3e3d7fb58fd17, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=quincy-c61472c, GIT_COMMIT=e83daab126629de4b83b70f7819e84da953b5b67, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-26T05:07:01.972 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@osd.7.service' 2024-04-26T05:07:02.014 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:07:02.014 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-26T05:07:02.015 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-26T05:07:02.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a 2024-04-26T05:07:02.352 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 systemd[1]: Stopping Ceph prometheus.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:07:02.606 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:07:02.606 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.352Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.353Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-26T05:07:02.607 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.353Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-26T05:07:02.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a[76288]: ts=2024-04-26T05:07:02.353Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-26T05:07:02.608 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 podman[100219]: 2024-04-26 05:07:02.362644337 +0000 UTC m=+0.116720519 container died c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:03.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 podman[100219]: 2024-04-26 05:07:02.743720944 +0000 UTC m=+0.497797158 container cleanup c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:03.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 bash[100219]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a 2024-04-26T05:07:03.106 INFO:journalctl@ceph.prometheus.a.smithi055.stdout:Apr 26 05:07:02 smithi055 podman[100233]: 2024-04-26 05:07:02.894795534 +0000 UTC m=+0.534051065 container remove c40205ac54525238ee03838d373d3ebaeec1be89ad299d205b47249035831ee6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:03.128 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e22c1684-0385-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-26T05:07:03.163 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-26T05:07:03.163 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-26T05:07:03.164 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e22c1684-0385-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:07:04.475 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:04.316Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:07:04.475 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:04.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi038.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.38:8443: connect: connection refused" 2024-04-26T05:07:04.475 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:04.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2024-04-26T05:07:04.970 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:04 smithi038 systemd[1]: Stopping Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:07:05.226 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:04 smithi038 ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a[82241]: ts=2024-04-26T05:07:04.969Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-26T05:07:05.226 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 podman[149225]: 2024-04-26 05:07:05.020186173 +0000 UTC m=+0.126262727 container died bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:05.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 podman[149225]: 2024-04-26 05:07:05.469795743 +0000 UTC m=+0.575872297 container cleanup bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:05.724 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 bash[149225]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a 2024-04-26T05:07:06.167 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 podman[149239]: 2024-04-26 05:07:05.74613177 +0000 UTC m=+0.727140149 container remove bd09134fac6ba31cf89dea3e93ab50531bbb319c0701b7e0f6e91fe157987d44 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:06.167 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 podman[149239]: 2024-04-26 05:07:05.821470627 +0000 UTC m=+0.802479006 volume remove da727872143e0fe95cc74eeace2ca10c862adb3b76f683def1bc393c4bbeda8a 2024-04-26T05:07:06.167 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-26T05:07:06.167 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 systemd[1]: Stopped Ceph alertmanager.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T05:07:06.167 INFO:journalctl@ceph.alertmanager.a.smithi038.stdout:Apr 26 05:07:05 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 2.441s CPU time. 2024-04-26T05:07:06.418 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:06 smithi038 systemd[1]: Stopping Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:07:06.724 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:06 smithi038 podman[149347]: 2024-04-26 05:07:06.418384228 +0000 UTC m=+0.108733249 container died f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:07.224 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:06 smithi038 podman[149347]: 2024-04-26 05:07:06.833306877 +0000 UTC m=+0.523655896 container cleanup f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:07.224 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:06 smithi038 bash[149347]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a 2024-04-26T05:07:07.224 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:07 smithi038 podman[149361]: 2024-04-26 05:07:07.042963848 +0000 UTC m=+0.625047251 container remove f3d04e42cf8f1e5f82297823142ea83e4cdee9ce41526bce066913cab6953c0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-26T05:07:07.224 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:07 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:07:07.705 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:07 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-26T05:07:07.705 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:07 smithi038 systemd[1]: Stopped Ceph node-exporter.a for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T05:07:07.705 INFO:journalctl@ceph.node-exporter.a.smithi038.stdout:Apr 26 05:07:07 smithi038 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.879s CPU time. 2024-04-26T05:07:14.875 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e22c1684-0385-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-26T05:07:16.606 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:16 smithi055 systemd[1]: Stopping Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968... 2024-04-26T05:07:16.924 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:16 smithi055 podman[100633]: 2024-04-26 05:07:16.659003549 +0000 UTC m=+0.106306452 container died 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:07:17.267 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 podman[100633]: 2024-04-26 05:07:17.023715642 +0000 UTC m=+0.471018553 container cleanup 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:07:17.267 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 bash[100633]: ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b 2024-04-26T05:07:17.575 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 podman[100645]: 2024-04-26 05:07:17.266886985 +0000 UTC m=+0.611812796 container remove 9087143d9886420fbe40a40fc9abffd76a59b346611b661ffd9f3c3474865c28 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-e22c1684-0385-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-26T05:07:17.576 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-26T05:07:17.576 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-26T05:07:17.576 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 systemd[1]: Stopped Ceph node-exporter.b for e22c1684-0385-11ef-bc93-c7b262605968. 2024-04-26T05:07:17.576 INFO:journalctl@ceph.node-exporter.b.smithi055.stdout:Apr 26 05:07:17 smithi055 systemd[1]: ceph-e22c1684-0385-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.561s CPU time. 2024-04-26T05:07:22.146 DEBUG:teuthology.orchestra.run.smithi038:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:07:22.179 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-26T05:07:22.207 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-26T05:07:22.209 DEBUG:teuthology.misc:Transferring archived files from smithi038:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/remote/smithi038/crash 2024-04-26T05:07:22.210 DEBUG:teuthology.orchestra.run.smithi038:> sudo tar c -f - -C /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:07:22.265 INFO:teuthology.orchestra.run.smithi038.stderr:tar: /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:07:22.265 INFO:teuthology.orchestra.run.smithi038.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:07:22.267 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/remote/smithi055/crash 2024-04-26T05:07:22.268 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/crash -- . 2024-04-26T05:07:22.304 INFO:teuthology.orchestra.run.smithi055.stderr:tar: /var/lib/ceph/e22c1684-0385-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-26T05:07:22.304 INFO:teuthology.orchestra.run.smithi055.stderr:tar: Error is not recoverable: exiting now 2024-04-26T05:07:22.305 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-26T05:07:22.305 DEBUG:teuthology.orchestra.run.smithi038:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-26T05:07:22.369 INFO:teuthology.orchestra.run.smithi038.stdout:1714106240.0348113 mon.a (mon.0) 508 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-26T05:07:22.370 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-26T05:07:22.370 INFO:tasks.cephadm:Compressing logs... 2024-04-26T05:07:22.370 DEBUG:teuthology.orchestra.run.smithi038:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:07:22.373 DEBUG:teuthology.orchestra.run.smithi055:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-26T05:07:22.397 INFO:teuthology.orchestra.run.smithi055.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-26T05:07:22.397 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:07:22.397 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:07:22.397 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-26T05:07:22.399 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-26T05:07:22.399 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-26T05:07:22.399 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-26T05:07:22.401 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.5.log: /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-26T05:07:22.401 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:07:22.402 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:07:22.402 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.cephadm.log: 84.2% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:07:22.402 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mds.foo.smithi055.cuaegn.log 2024-04-26T05:07:22.402 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-volume.log: 87.3% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:07:22.404 INFO:teuthology.orchestra.run.smithi055.stderr: 89.4% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:07:22.404 INFO:teuthology.orchestra.run.smithi055.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-26T05:07:22.404 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mds.foo.smithi055.cuaegn.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:07:22.404 INFO:teuthology.orchestra.run.smithi055.stderr: 62.5% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mds.foo.smithi055.cuaegn.log.gz 2024-04-26T05:07:22.406 INFO:teuthology.orchestra.run.smithi038.stderr:find: ‘/var/log/rbd-target-api’: No such file or directorygzip 2024-04-26T05:07:22.406 INFO:teuthology.orchestra.run.smithi038.stderr: -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mds.foo.smithi038.qsffiu.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:gzip -5/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mds.foo.smithi038.qsffiu.log: --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log: 81.9% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mds.foo.smithi038.qsffiu.log.gz 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.2.log: /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-26T05:07:22.407 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-26T05:07:22.408 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-26T05:07:22.408 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-26T05:07:22.409 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-26T05:07:22.409 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-26T05:07:22.409 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-26T05:07:22.410 INFO:teuthology.orchestra.run.smithi038.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 92.8% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-26T05:07:22.482 INFO:teuthology.orchestra.run.smithi038.stderr: 90.6% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-26T05:07:22.546 INFO:teuthology.orchestra.run.smithi055.stderr:/var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 91.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:07:22.569 INFO:teuthology.orchestra.run.smithi055.stderr: 91.4% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:07:22.715 INFO:teuthology.orchestra.run.smithi038.stderr: 93.6% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-26T05:07:22.778 INFO:teuthology.orchestra.run.smithi055.stderr: 89.6% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-26T05:07:22.832 INFO:teuthology.orchestra.run.smithi038.stderr: 91.1% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-26T05:07:22.843 INFO:teuthology.orchestra.run.smithi055.stderr: 91.9% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-26T05:07:23.565 INFO:teuthology.orchestra.run.smithi038.stderr: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-26T05:07:24.239 INFO:teuthology.orchestra.run.smithi038.stderr: 88.7% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-26T05:07:26.190 INFO:teuthology.orchestra.run.smithi038.stderr: 90.2% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-26T05:07:27.746 INFO:teuthology.orchestra.run.smithi038.stderr: 91.6% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-26T05:08:02.088 INFO:teuthology.orchestra.run.smithi055.stderr: 93.7% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-26T05:08:03.292 INFO:teuthology.orchestra.run.smithi055.stderr: 93.7% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-26T05:08:04.323 INFO:teuthology.orchestra.run.smithi055.stderr: 93.6% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-26T05:08:23.621 INFO:teuthology.orchestra.run.smithi055.stderr: 93.3% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-26T05:08:23.623 INFO:teuthology.orchestra.run.smithi055.stderr: 2024-04-26T05:08:23.623 INFO:teuthology.orchestra.run.smithi055.stderr:real 1m1.236s 2024-04-26T05:08:23.623 INFO:teuthology.orchestra.run.smithi055.stderr:user 2m57.350s 2024-04-26T05:08:23.624 INFO:teuthology.orchestra.run.smithi055.stderr:sys 0m6.171s 2024-04-26T05:08:39.167 INFO:teuthology.orchestra.run.smithi038.stderr: 93.1% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-26T05:09:04.804 INFO:teuthology.orchestra.run.smithi038.stderr: 93.5% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-26T05:09:16.423 INFO:teuthology.orchestra.run.smithi038.stderr: 93.4% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-26T05:09:24.033 INFO:teuthology.orchestra.run.smithi038.stderr: 93.3% -- replaced with /var/log/ceph/e22c1684-0385-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-26T05:09:24.035 INFO:teuthology.orchestra.run.smithi038.stderr: 2024-04-26T05:09:24.035 INFO:teuthology.orchestra.run.smithi038.stderr:real 2m1.647s 2024-04-26T05:09:24.035 INFO:teuthology.orchestra.run.smithi038.stderr:user 3m57.411s 2024-04-26T05:09:24.035 INFO:teuthology.orchestra.run.smithi038.stderr:sys 0m8.630s 2024-04-26T05:09:24.036 INFO:tasks.cephadm:Archiving logs... 2024-04-26T05:09:24.036 DEBUG:teuthology.misc:Transferring archived files from smithi038:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/remote/smithi038/log 2024-04-26T05:09:24.037 DEBUG:teuthology.orchestra.run.smithi038:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:09:59.248 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/remote/smithi055/log 2024-04-26T05:09:59.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-26T05:10:25.540 INFO:tasks.cephadm:Removing cluster... 2024-04-26T05:10:25.540 DEBUG:teuthology.orchestra.run.smithi038:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e22c1684-0385-11ef-bc93-c7b262605968 --force 2024-04-26T05:10:26.353 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e22c1684-0385-11ef-bc93-c7b262605968 --force 2024-04-26T05:10:27.128 INFO:tasks.cephadm:Removing cephadm ... 2024-04-26T05:10:27.129 DEBUG:teuthology.orchestra.run.smithi038:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:10:27.148 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-26T05:10:27.163 INFO:tasks.cephadm:Teardown complete 2024-04-26T05:10:27.164 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-26T05:10:27.174 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-26T05:10:27.174 DEBUG:teuthology.orchestra.run.smithi038:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:10:27.191 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-26T05:10:27.280 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-26T05:10:27.289 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-26T05:10:27.290 DEBUG:teuthology.orchestra.run.smithi038:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:10:27.292 DEBUG:teuthology.orchestra.run.smithi055:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-26T05:10:27.307 INFO:teuthology.orchestra.run.smithi038.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:10:27.307 INFO:teuthology.orchestra.run.smithi055.stderr:bash: line 1: ntpq: command not found 2024-04-26T05:10:27.311 INFO:teuthology.orchestra.run.smithi055.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:10:27.311 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================== 2024-04-26T05:10:27.311 INFO:teuthology.orchestra.run.smithi055.stdout:^* hv01.front.sepia.ceph.com 3 6 377 3 +984us[ +969us] +/- 54ms 2024-04-26T05:10:27.312 INFO:teuthology.orchestra.run.smithi055.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 9 -1311us[-1327us] +/- 73ms 2024-04-26T05:10:27.312 INFO:teuthology.orchestra.run.smithi055.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 6 -56us[ -72us] +/- 68ms 2024-04-26T05:10:27.312 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:10:27.316 INFO:teuthology.orchestra.run.smithi038.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-26T05:10:27.316 INFO:teuthology.orchestra.run.smithi038.stdout:=============================================================================== 2024-04-26T05:10:27.316 INFO:teuthology.orchestra.run.smithi038.stdout:^* hv01.front.sepia.ceph.com 3 6 377 7 +981us[ +979us] +/- 54ms 2024-04-26T05:10:27.316 INFO:teuthology.orchestra.run.smithi038.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 5 -1317us[-1317us] +/- 73ms 2024-04-26T05:10:27.317 INFO:teuthology.orchestra.run.smithi038.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 5 -55us[ -55us] +/- 68ms 2024-04-26T05:10:27.317 INFO:teuthology.orchestra.run.smithi038.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-26T05:10:27.318 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-26T05:10:27.328 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-26T05:10:27.329 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-26T05:10:27.365 DEBUG:teuthology.orchestra.run.smithi038:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:10:27.460 DEBUG:teuthology.orchestra.run.smithi055:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-26T05:10:27.532 DEBUG:teuthology.orchestra.run.smithi038:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:10:27.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:10:27.562 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-26T05:10:27.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:10:27.589 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-26T05:10:27.599 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-26T05:10:27.665 INFO:teuthology.task.internal:Duration was 2691.017155 seconds 2024-04-26T05:10:27.666 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-26T05:10:27.748 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-26T05:10:27.749 DEBUG:teuthology.orchestra.run.smithi038:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:10:27.751 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-26T05:10:27.790 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:10:27.827 INFO:teuthology.orchestra.run.smithi038.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-26T05:10:28.272 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-26T05:10:28.272 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi038.front.sepia.ceph.com 2024-04-26T05:10:28.272 DEBUG:teuthology.orchestra.run.smithi038:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:10:28.296 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi055.front.sepia.ceph.com 2024-04-26T05:10:28.296 DEBUG:teuthology.orchestra.run.smithi055:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-26T05:10:28.323 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-26T05:10:28.323 DEBUG:teuthology.orchestra.run.smithi038:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:10:28.339 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-26T05:10:28.390 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-26T05:10:28.390 DEBUG:teuthology.orchestra.run.smithi038:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:10:28.408 DEBUG:teuthology.orchestra.run.smithi055:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-26T05:10:28.899 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-26T05:10:28.910 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-26T05:10:28.910 DEBUG:teuthology.orchestra.run.smithi038:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:10:28.948 DEBUG:teuthology.orchestra.run.smithi055:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-26T05:10:28.974 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-26T05:10:28.983 DEBUG:teuthology.orchestra.run.smithi038:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:10:28.991 DEBUG:teuthology.orchestra.run.smithi055:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:10:29.011 INFO:teuthology.orchestra.run.smithi038.stdout:kernel.core_pattern = core 2024-04-26T05:10:29.037 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = core 2024-04-26T05:10:29.096 DEBUG:teuthology.orchestra.run.smithi038:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:10:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:10:29.151 DEBUG:teuthology.orchestra.run.smithi055:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-26T05:10:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-26T05:10:29.165 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-26T05:10:29.176 INFO:teuthology.task.internal:Transferring archived files... 2024-04-26T05:10:29.178 DEBUG:teuthology.misc:Transferring archived files from smithi038:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/remote/smithi038 2024-04-26T05:10:29.178 DEBUG:teuthology.orchestra.run.smithi038:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:10:29.237 DEBUG:teuthology.misc:Transferring archived files from smithi055:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562/remote/smithi055 2024-04-26T05:10:29.237 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-26T05:10:29.275 INFO:teuthology.task.internal:Removing archive directory... 2024-04-26T05:10:29.275 DEBUG:teuthology.orchestra.run.smithi038:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:10:29.278 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-26T05:10:29.324 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-26T05:10:29.336 INFO:teuthology.task.internal:Not uploading archives. 2024-04-26T05:10:29.336 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-26T05:10:29.366 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-26T05:10:29.366 DEBUG:teuthology.orchestra.run.smithi038:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:10:29.369 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-26T05:10:29.381 INFO:teuthology.orchestra.run.smithi038.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:10 /home/ubuntu/cephtest 2024-04-26T05:10:29.383 INFO:teuthology.orchestra.run.smithi055.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 26 05:10 /home/ubuntu/cephtest 2024-04-26T05:10:29.384 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-26T05:10:29.416 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-26T05:10:29.516 INFO:teuthology.nuke:Checking targets against current locks 2024-04-26T05:10:29.542 DEBUG:teuthology.nuke:shortname: smithi038 2024-04-26T05:10:29.543 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:10:29.567 DEBUG:teuthology.nuke:shortname: smithi055 2024-04-26T05:10:29.568 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-26T05:10:29.571 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi038.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:20:17.323916', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAacMjVVYK871bpzS385fVCDFelFLOfZinui4I3jyXCbdZW7ex1QlubBdLTLS8M+N6YfN8ezerdLtFpgVkrWcpg='} 2024-04-26T05:10:29.594 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-26_02:08:13-upgrade-squid-distro-default-smithi/7674562', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-26 04:20:17.325037', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPnZbdTwWwE5etQ8bkwSIvLYlHKEecZ0zgXcLkBgyZngtxPTJd5Z33X0vk0RiST7WJRAWhx9LHhwLJjVniEFw0c='} 2024-04-26T05:10:29.618 INFO:teuthology.orchestra.console.smithi038:Power off 2024-04-26T05:10:29.618 DEBUG:teuthology.orchestra.console.smithi038:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:10:29.642 DEBUG:teuthology.orchestra.console.smithi038:power off output: Chassis Power Control: Down/Off 2024-04-26T05:10:29.642 DEBUG:teuthology.orchestra.console.smithi038:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:29.657 INFO:teuthology.orchestra.console.smithi055:Power off 2024-04-26T05:10:29.658 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-26T05:10:29.666 DEBUG:teuthology.orchestra.console.smithi038:check power output: Chassis Power is on 2024-04-26T05:10:29.679 DEBUG:teuthology.orchestra.console.smithi055:power off output: Chassis Power Control: Down/Off 2024-04-26T05:10:29.680 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:29.702 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2024-04-26T05:10:33.667 DEBUG:teuthology.orchestra.console.smithi038:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:33.704 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:33.778 DEBUG:teuthology.orchestra.console.smithi038:check power output: Chassis Power is on 2024-04-26T05:10:33.814 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2024-04-26T05:10:37.782 DEBUG:teuthology.orchestra.console.smithi038:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:37.815 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:37.895 DEBUG:teuthology.orchestra.console.smithi038:check power output: Chassis Power is on 2024-04-26T05:10:37.925 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is off 2024-04-26T05:10:38.026 INFO:teuthology.orchestra.console.smithi055:Power off completed 2024-04-26T05:10:41.900 DEBUG:teuthology.orchestra.console.smithi038:pexpect command: ipmitool -H smithi038.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-26T05:10:42.010 DEBUG:teuthology.orchestra.console.smithi038:check power output: Chassis Power is off 2024-04-26T05:10:42.111 INFO:teuthology.orchestra.console.smithi038:Power off completed 2024-04-26T05:10:42.216 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 2691.017154932022 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=81137905cac347419c6011b00041bde1 status: fail success: false 2024-04-26T05:10:42.216 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-26T05:10:42.302 INFO:teuthology.run:FAIL